1. Install and Run Locally
Create a virtual environment, install requirements, install Tesseract and FFmpeg, add environment values, then run python app.py on port 5000.
Build a Flask chatbot that receives WhatsApp text, receipt images, and voice notes through Twilio, extracts item and amount, categorizes spending, saves to Google Sheets, and replies with friendly summaries.
Follow these steps in order. The app runs locally first, then ngrok exposes it to Twilio.
Create a virtual environment, install requirements, install Tesseract and FFmpeg, add environment values, then run python app.py on port 5000.
Enable Google Sheets API and Drive API, create a service account, download credentials.json, create the sheet, and share it with the service account email.
Use the Twilio WhatsApp Sandbox, expose Flask with ngrok, and set the incoming message webhook to your HTTPS ngrok URL followed by /webhook.
Send Milk 50, bill images, and voice notes. Use total, today, summary, and weekly to confirm Google Sheets reads work.
Copy each file into the matching path shown below.
import os
from flask import Flask, request
from twilio.twiml.messaging_response import MessagingResponse
from twilio.rest import Client
from dotenv import load_dotenv
from parser import parse_expense, detect_command
from sheets import add_expense, get_total, get_today_expenses, get_monthly_summary, get_weekly_summary
from ocr import extract_text_from_image
from speech import transcribe_audio
load_dotenv()
app = Flask(__name__)
TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)
def download_twilio_media(media_url, file_extension):
"""Download image/audio sent to WhatsApp through Twilio."""
response = client.request("GET", media_url)
os.makedirs("downloads", exist_ok=True)
file_path = os.path.join("downloads", f"media_{os.urandom(6).hex()}.{file_extension}")
with open(file_path, "wb") as media_file:
media_file.write(response.content)
return file_path
def create_reply(message):
"""Create a Twilio WhatsApp reply."""
response = MessagingResponse()
response.message(message)
return str(response)
@app.route("/", methods=["GET"])
def home():
return "WhatsApp Expense Tracker Bot is running. Use /webhook for Twilio.", 200
@app.route("/webhook", methods=["POST"])
def webhook():
"""Main Twilio webhook for incoming WhatsApp messages."""
incoming_text = (request.form.get("Body") or "").strip()
media_count = int(request.form.get("NumMedia", 0))
try:
# 1. Handle text commands first.
command = detect_command(incoming_text)
if command == "total":
total = get_total()
return create_reply(f"💰 Total expenses: ₹{total}")
if command == "today":
rows, total = get_today_expenses()
if not rows:
return create_reply("📭 No expenses recorded today.")
lines = [f"📅 Today's expenses: ₹{total}"]
for row in rows:
lines.append(f"• {row['Item']} - ₹{row['Amount']} ({row['Category']})")
return create_reply("
".join(lines))
if command == "summary":
month_name, total, category_totals = get_monthly_summary()
lines = [f"📊 {month_name} summary: ₹{total}"]
for category, amount in category_totals.items():
lines.append(f"• {category}: ₹{amount}")
return create_reply("
".join(lines))
if command == "weekly":
total = get_weekly_summary()
return create_reply(f"🗓️ This week's expenses: ₹{total}")
extracted_text = incoming_text
# 2. Handle media: image or audio.
if media_count > 0:
media_url = request.form.get("MediaUrl0")
content_type = request.form.get("MediaContentType0", "")
if content_type.startswith("image"):
image_path = download_twilio_media(media_url, "jpg")
extracted_text = extract_text_from_image(image_path)
elif content_type.startswith("audio"):
audio_path = download_twilio_media(media_url, "ogg")
extracted_text = transcribe_audio(audio_path)
else:
return create_reply("❌ Unsupported media type. Please send text, image, or voice note.")
# 3. Parse extracted text into expense fields.
expense = parse_expense(extracted_text)
if not expense:
return create_reply("❌ Could not detect amount. Please try again.")
# 4. Store in Google Sheets.
add_expense(expense)
reply = (
"✅ Expense added:
"
f"Item: {expense['item']}
"
f"Amount: ₹{expense['amount']}
"
f"Category: {expense['category'].title()}"
)
return create_reply(reply)
except Exception as error:
print("Webhook error:", error)
return create_reply("⚠️ Something went wrong. Please try again in a moment.")
if __name__ == "__main__":
app.run(debug=True, port=5000)
Use these after Flask, ngrok, Twilio, and Google Sheets are connected.
If the parser cannot find a numeric amount, the webhook returns: ❌ Could not detect amount. Please try again.