506 lines
20 KiB
Python
Executable File
506 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Audita mensagens que existem no Chatwoot mas não aparecem no ClientFlow.
|
|
|
|
Caso de uso: ClientFlow esteve em baixo e o Chatwoot não conseguiu entregar
|
|
webhooks. Nessa situação não existe raw_event para auditar; por isso este script
|
|
lê conversas/mensagens diretamente da API do Chatwoot e compara cada mensagem
|
|
inbound pública com raw_events/messages/communications no PostgreSQL.
|
|
|
|
Uso seguro:
|
|
PYTHONPATH=. python scripts/audit_chatwoot_remote_missing_messages.py --hours 72
|
|
PYTHONPATH=. python scripts/audit_chatwoot_remote_missing_messages.py --hours 72 --include-ok
|
|
PYTHONPATH=. python scripts/audit_chatwoot_remote_missing_messages.py --hours 72 --post-missing --limit 25
|
|
|
|
Variáveis necessárias:
|
|
CHATWOOT_BASE_URL
|
|
CHATWOOT_ACCOUNT_ID
|
|
CHATWOOT_API_TOKEN
|
|
DATABASE_URL
|
|
|
|
Para --post-missing também:
|
|
CLIENTFLOW_WEBHOOK_SECRET
|
|
CLIENTFLOW_WEBHOOK_URL opcional; por defeito http://127.0.0.1:8020/webhooks/chatwoot
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
os.chdir(PROJECT_ROOT)
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import engine
|
|
|
|
OUT_CSV = Path("/tmp/clientflow_chatwoot_remote_missing_messages.csv")
|
|
OUT_MD = Path("/tmp/clientflow_chatwoot_remote_missing_messages.md")
|
|
|
|
|
|
def env(name: str, default: str = "") -> str:
|
|
return os.getenv(name, default).strip()
|
|
|
|
|
|
CHATWOOT_BASE_URL = env("CHATWOOT_BASE_URL").rstrip("/")
|
|
CHATWOOT_ACCOUNT_ID = env("CHATWOOT_ACCOUNT_ID")
|
|
CHATWOOT_API_TOKEN = env("CHATWOOT_API_TOKEN")
|
|
CLIENTFLOW_WEBHOOK_SECRET = env("CLIENTFLOW_WEBHOOK_SECRET")
|
|
CLIENTFLOW_WEBHOOK_URL = env("CLIENTFLOW_WEBHOOK_URL", "http://127.0.0.1:8020/webhooks/chatwoot")
|
|
|
|
|
|
@dataclass
|
|
class RemoteMessage:
|
|
conversation: dict[str, Any]
|
|
message: dict[str, Any]
|
|
recent_context: list[dict[str, Any]]
|
|
|
|
|
|
def clean(value: Any, limit: int = 240) -> str:
|
|
txt = " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split())
|
|
return txt if len(txt) <= limit else txt[: limit - 1].rstrip() + "…"
|
|
|
|
|
|
def ts_to_datetime(value: Any) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
if isinstance(value, (int, float)):
|
|
return datetime.fromtimestamp(float(value), tz=timezone.utc)
|
|
s = str(value).strip().replace("Z", "+00:00")
|
|
if not s:
|
|
return None
|
|
return datetime.fromisoformat(s).astimezone(timezone.utc)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def dt_to_display(value: Any) -> str:
|
|
dt = ts_to_datetime(value)
|
|
return dt.isoformat() if dt else str(value or "")
|
|
|
|
|
|
def message_created_at(message: dict[str, Any]) -> datetime:
|
|
return ts_to_datetime(message.get("created_at")) or datetime.fromtimestamp(0, tz=timezone.utc)
|
|
|
|
|
|
def conversation_last_activity(conversation: dict[str, Any]) -> datetime | None:
|
|
for key in ["last_activity_at", "updated_at", "created_at"]:
|
|
dt = ts_to_datetime(conversation.get(key))
|
|
if dt:
|
|
return dt
|
|
return None
|
|
|
|
|
|
def request_json(method: str, url: str, *, body: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> dict[str, Any]:
|
|
data = None
|
|
final_headers = dict(headers or {})
|
|
if body is not None:
|
|
data = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
final_headers["Content-Type"] = "application/json"
|
|
|
|
req = urllib.request.Request(url, data=data, headers=final_headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=45) as resp:
|
|
raw = resp.read().decode("utf-8", errors="replace")
|
|
try:
|
|
parsed: Any = json.loads(raw) if raw else {}
|
|
except json.JSONDecodeError:
|
|
parsed = {}
|
|
return {"ok": 200 <= resp.status < 300, "status": resp.status, "json": parsed, "raw": raw}
|
|
except urllib.error.HTTPError as e:
|
|
raw = e.read().decode("utf-8", errors="replace")
|
|
return {"ok": False, "status": e.code, "json": None, "raw": raw}
|
|
except urllib.error.URLError as e:
|
|
return {"ok": False, "status": 0, "json": None, "raw": str(e)}
|
|
|
|
|
|
def chatwoot_headers() -> dict[str, str]:
|
|
return {"api_access_token": CHATWOOT_API_TOKEN, "Accept": "application/json"}
|
|
|
|
|
|
def payload_list(data: Any) -> list[dict[str, Any]]:
|
|
if isinstance(data, list):
|
|
return [item for item in data if isinstance(item, dict)]
|
|
if not isinstance(data, dict):
|
|
return []
|
|
|
|
candidates = [
|
|
data.get("payload"),
|
|
data.get("data", {}).get("payload") if isinstance(data.get("data"), dict) else None,
|
|
data.get("data"),
|
|
data.get("messages"),
|
|
]
|
|
for item in candidates:
|
|
if isinstance(item, list):
|
|
return [row for row in item if isinstance(row, dict)]
|
|
return []
|
|
|
|
|
|
def fetch_conversations(status: str, *, max_pages: int) -> list[dict[str, Any]]:
|
|
all_items: list[dict[str, Any]] = []
|
|
for page in range(1, max_pages + 1):
|
|
query = urllib.parse.urlencode({"status": status, "page": page})
|
|
url = f"{CHATWOOT_BASE_URL}/api/v1/accounts/{CHATWOOT_ACCOUNT_ID}/conversations?{query}"
|
|
result = request_json("GET", url, headers=chatwoot_headers())
|
|
if not result["ok"]:
|
|
print(f"ERRO Chatwoot conversations status={status} page={page}: {result['status']} {result['raw'][:300]}")
|
|
break
|
|
items = payload_list(result["json"])
|
|
if not items:
|
|
break
|
|
all_items.extend(items)
|
|
if len(items) < 10:
|
|
break
|
|
return all_items
|
|
|
|
|
|
def fetch_messages(conversation_id: str) -> list[dict[str, Any]]:
|
|
url = f"{CHATWOOT_BASE_URL}/api/v1/accounts/{CHATWOOT_ACCOUNT_ID}/conversations/{conversation_id}/messages"
|
|
result = request_json("GET", url, headers=chatwoot_headers())
|
|
if not result["ok"]:
|
|
print(f"ERRO Chatwoot messages conversation={conversation_id}: {result['status']} {result['raw'][:300]}")
|
|
return []
|
|
return payload_list(result["json"])
|
|
|
|
|
|
def is_incoming(message: dict[str, Any]) -> bool:
|
|
mt = message.get("message_type")
|
|
return mt == "incoming" or mt == 0 or str(mt).lower() == "incoming"
|
|
|
|
|
|
def message_content(message: dict[str, Any]) -> str:
|
|
return str(message.get("content") or message.get("text") or message.get("body") or "").strip()
|
|
|
|
|
|
def message_id(message: dict[str, Any]) -> str:
|
|
return str(message.get("id") or message.get("message_id") or "").strip()
|
|
|
|
|
|
def sender_from_conversation(conversation: dict[str, Any], message: dict[str, Any]) -> dict[str, Any]:
|
|
sender: dict[str, Any] = {}
|
|
meta = conversation.get("meta") or {}
|
|
if isinstance(meta, dict) and isinstance(meta.get("sender"), dict):
|
|
sender.update(meta.get("sender") or {})
|
|
if isinstance(conversation.get("contact"), dict):
|
|
sender.update({k: v for k, v in conversation["contact"].items() if v is not None})
|
|
if isinstance(message.get("sender"), dict):
|
|
sender.update({k: v for k, v in message["sender"].items() if v is not None})
|
|
return sender
|
|
|
|
|
|
def iter_remote_messages(
|
|
*,
|
|
statuses: Iterable[str],
|
|
cutoff: datetime,
|
|
max_pages: int,
|
|
max_context_messages: int,
|
|
) -> list[RemoteMessage]:
|
|
seen_conversations: set[str] = set()
|
|
selected: list[RemoteMessage] = []
|
|
|
|
for status in statuses:
|
|
conversations = fetch_conversations(status, max_pages=max_pages)
|
|
print(f"--- status={status} conversations={len(conversations)}")
|
|
for conv in conversations:
|
|
conv_id = str(conv.get("id") or "").strip()
|
|
if not conv_id or conv_id in seen_conversations:
|
|
continue
|
|
seen_conversations.add(conv_id)
|
|
|
|
last_activity = conversation_last_activity(conv)
|
|
if last_activity and last_activity < cutoff:
|
|
continue
|
|
|
|
messages = fetch_messages(conv_id)
|
|
messages.sort(key=message_created_at)
|
|
public_messages = [m for m in messages if not m.get("private") and message_content(m)]
|
|
|
|
for index, msg in enumerate(public_messages):
|
|
if not is_incoming(msg):
|
|
continue
|
|
if message_created_at(msg) < cutoff:
|
|
continue
|
|
mid = message_id(msg)
|
|
if not mid:
|
|
continue
|
|
start = max(0, index - max_context_messages)
|
|
context = public_messages[start : index + 1]
|
|
selected.append(RemoteMessage(conversation=conv, message=msg, recent_context=context))
|
|
|
|
selected.sort(key=lambda item: message_created_at(item.message))
|
|
return selected
|
|
|
|
|
|
def load_clientflow_status(source_event_ids: list[str]) -> dict[str, dict[str, Any]]:
|
|
if not source_event_ids:
|
|
return {}
|
|
with engine.begin() as conn:
|
|
rows = conn.execute(text("""
|
|
WITH ids AS (
|
|
SELECT unnest(CAST(:ids AS text[])) AS source_event_id
|
|
)
|
|
SELECT
|
|
ids.source_event_id,
|
|
re.id::text AS raw_event_id,
|
|
re.created_at AS raw_event_created_at,
|
|
re.processed,
|
|
re.ignored,
|
|
re.processing_error,
|
|
re.message_id::text AS raw_event_message_id,
|
|
re.action_run_id::text AS raw_event_action_run_id,
|
|
m.id::text AS message_id,
|
|
c.id::text AS communication_id,
|
|
o.id::text AS opportunity_id
|
|
FROM ids
|
|
LEFT JOIN raw_events re
|
|
ON re.source_system = 'chatwoot'
|
|
AND re.source_event_id = ids.source_event_id
|
|
LEFT JOIN messages m
|
|
ON (
|
|
(re.message_id IS NOT NULL AND m.id = re.message_id)
|
|
OR (re.id IS NOT NULL AND m.raw_event_id = re.id)
|
|
OR (m.source_system = 'chatwoot' AND m.source_event_id = ids.source_event_id)
|
|
)
|
|
LEFT JOIN communications c
|
|
ON c.source_system = 'chatwoot'
|
|
AND c.source_message_id = ids.source_event_id
|
|
LEFT JOIN opportunities o
|
|
ON o.conversation_id = COALESCE(re.conversation_id, m.conversation_id)
|
|
"""), {"ids": source_event_ids}).mappings().all()
|
|
return {str(row["source_event_id"]): dict(row) for row in rows}
|
|
|
|
|
|
def classify(row: dict[str, Any] | None) -> str:
|
|
row = row or {}
|
|
visible_message_id = row.get("message_id") or row.get("raw_event_message_id")
|
|
if not row.get("raw_event_id") and not visible_message_id and not row.get("communication_id"):
|
|
return "MISSING_IN_CLIENTFLOW"
|
|
if row.get("processing_error"):
|
|
return "RAW_PROCESSING_ERROR"
|
|
if row.get("raw_event_id") and not row.get("processed") and not row.get("ignored"):
|
|
return "RAW_PENDING"
|
|
if row.get("ignored"):
|
|
return "RAW_IGNORED"
|
|
if row.get("processed") and not visible_message_id and not row.get("communication_id"):
|
|
return "PROCESSED_BUT_NOT_VISIBLE"
|
|
if not row.get("opportunity_id"):
|
|
return "VISIBLE_BUT_UNLINKED_CONVERSATION"
|
|
return "OK"
|
|
|
|
|
|
def build_webhook_payload(remote: RemoteMessage) -> dict[str, Any]:
|
|
conv = remote.conversation
|
|
msg = remote.message
|
|
conv_id = str(conv.get("id") or msg.get("conversation_id") or "")
|
|
sender = sender_from_conversation(conv, msg)
|
|
contact_id = str(sender.get("id") or conv.get("contact_id") or conv.get("contact", {}).get("id") or msg.get("contact_id") or "")
|
|
|
|
normalized_message = dict(msg)
|
|
normalized_message["id"] = message_id(msg)
|
|
normalized_message["content"] = message_content(msg)
|
|
normalized_message["message_type"] = "incoming"
|
|
normalized_message["conversation_id"] = conv_id
|
|
normalized_message["sender"] = sender
|
|
normalized_message["private"] = bool(msg.get("private"))
|
|
|
|
conversation_payload = dict(conv)
|
|
conversation_payload["id"] = conv_id
|
|
conversation_payload["messages"] = remote.recent_context
|
|
conversation_payload["contact"] = conv.get("contact") if isinstance(conv.get("contact"), dict) else sender
|
|
meta = conversation_payload.get("meta") if isinstance(conversation_payload.get("meta"), dict) else {}
|
|
meta = dict(meta)
|
|
meta.setdefault("sender", sender)
|
|
conversation_payload["meta"] = meta
|
|
|
|
return {
|
|
"event": "message_created",
|
|
"id": normalized_message["id"],
|
|
"message_id": normalized_message["id"],
|
|
"conversation_id": conv_id,
|
|
"contact_id": contact_id,
|
|
"content": normalized_message["content"],
|
|
"message_type": "incoming",
|
|
"private": bool(msg.get("private")),
|
|
"message": normalized_message,
|
|
"conversation": conversation_payload,
|
|
"sender": sender,
|
|
"backfill": {
|
|
"source": "chatwoot_remote_missing_messages",
|
|
"reason": "clientflow_down_or_webhook_not_delivered",
|
|
"audited_at": datetime.now(timezone.utc).isoformat(),
|
|
},
|
|
}
|
|
|
|
|
|
def sign_body(body_raw: str) -> dict[str, str]:
|
|
ts = str(int(time.time()))
|
|
msg = ts.encode("utf-8") + b"." + body_raw.encode("utf-8")
|
|
sig = "sha256=" + hmac.new(CLIENTFLOW_WEBHOOK_SECRET.encode("utf-8"), msg, hashlib.sha256).hexdigest()
|
|
return {"Content-Type": "application/json", "X-Chatwoot-Timestamp": ts, "X-Chatwoot-Signature": sig}
|
|
|
|
|
|
def post_to_clientflow(payload: dict[str, Any]) -> dict[str, Any]:
|
|
body_raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
|
headers = sign_body(body_raw)
|
|
req = urllib.request.Request(CLIENTFLOW_WEBHOOK_URL, data=body_raw.encode("utf-8"), headers=headers, method="POST")
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
raw = resp.read().decode("utf-8", errors="replace")
|
|
return {"ok": 200 <= resp.status < 300, "status": resp.status, "raw": raw}
|
|
except urllib.error.HTTPError as e:
|
|
raw = e.read().decode("utf-8", errors="replace")
|
|
return {"ok": False, "status": e.code, "raw": raw}
|
|
except urllib.error.URLError as e:
|
|
return {"ok": False, "status": 0, "raw": str(e)}
|
|
|
|
|
|
def write_reports(rows: list[dict[str, Any]], counts: dict[str, int], *, hours: int) -> None:
|
|
fieldnames = list(rows[0].keys()) if rows else [
|
|
"status", "chatwoot_created_at", "conversation_id", "source_event_id", "email", "content",
|
|
"raw_event_id", "processed", "ignored", "processing_error", "message_id", "communication_id", "opportunity_id", "post_status",
|
|
]
|
|
with OUT_CSV.open("w", encoding="utf-8", newline="") as f:
|
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
with OUT_MD.open("w", encoding="utf-8") as f:
|
|
f.write("# Auditoria remota Chatwoot → ClientFlow\n\n")
|
|
f.write(f"Janela: últimas {hours} horas\n\n")
|
|
f.write("## Resumo\n\n")
|
|
if not counts:
|
|
f.write("- OK: sem gaps inbound encontrados\n")
|
|
for status, total in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
|
|
f.write(f"- {status}: {total}\n")
|
|
f.write("\n## Casos\n\n")
|
|
for row in rows[:300]:
|
|
f.write(f"### {row['status']} · conv #{row['conversation_id']} · msg {row['source_event_id']}\n\n")
|
|
f.write(f"- Data Chatwoot: {row['chatwoot_created_at']}\n")
|
|
f.write(f"- Email: {row['email'] or '-'}\n")
|
|
f.write(f"- Raw event: {row['raw_event_id'] or '-'}\n")
|
|
f.write(f"- Erro: {row['processing_error'] or '-'}\n")
|
|
if row.get("post_status"):
|
|
f.write(f"- Post recovery: {row['post_status']}\n")
|
|
f.write(f"- Excerto: {row['content'] or '-'}\n\n")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--hours", type=int, default=int(env("CHATWOOT_AUDIT_HOURS", "168")))
|
|
parser.add_argument("--statuses", default=env("CHATWOOT_AUDIT_STATUSES", "open,pending,resolved,snoozed"))
|
|
parser.add_argument("--max-pages", type=int, default=int(env("CHATWOOT_AUDIT_MAX_PAGES", "50")))
|
|
parser.add_argument("--include-ok", action="store_true")
|
|
parser.add_argument("--post-missing", action="store_true", help="Reenvia para /webhooks/chatwoot apenas MISSING_IN_CLIENTFLOW")
|
|
parser.add_argument("--limit", type=int, default=0, help="Limita mensagens a auditar/repostar; 0 = sem limite")
|
|
parser.add_argument("--max-context-messages", type=int, default=5)
|
|
args = parser.parse_args()
|
|
|
|
required = {
|
|
"CHATWOOT_BASE_URL": CHATWOOT_BASE_URL,
|
|
"CHATWOOT_ACCOUNT_ID": CHATWOOT_ACCOUNT_ID,
|
|
"CHATWOOT_API_TOKEN": CHATWOOT_API_TOKEN,
|
|
}
|
|
if args.post_missing:
|
|
required["CLIENTFLOW_WEBHOOK_SECRET"] = CLIENTFLOW_WEBHOOK_SECRET
|
|
missing = [k for k, v in required.items() if not v]
|
|
if missing:
|
|
raise SystemExit(f"Faltam variáveis: {', '.join(missing)}")
|
|
|
|
statuses = [s.strip() for s in str(args.statuses or "").split(",") if s.strip()]
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=int(args.hours))
|
|
|
|
print(f"CHATWOOT_AUDIT_HOURS={args.hours}")
|
|
print(f"CHATWOOT_AUDIT_STATUSES={statuses}")
|
|
print(f"CHATWOOT_AUDIT_MAX_PAGES={args.max_pages}")
|
|
print(f"CUTOFF={cutoff.isoformat()}")
|
|
|
|
remote_messages = iter_remote_messages(
|
|
statuses=statuses,
|
|
cutoff=cutoff,
|
|
max_pages=int(args.max_pages),
|
|
max_context_messages=max(1, int(args.max_context_messages)),
|
|
)
|
|
if args.limit and args.limit > 0:
|
|
remote_messages = remote_messages[: int(args.limit)]
|
|
|
|
ids = [message_id(item.message) for item in remote_messages]
|
|
status_map = load_clientflow_status(ids)
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
counts: dict[str, int] = {}
|
|
posted = 0
|
|
failed = 0
|
|
|
|
for item in remote_messages:
|
|
msg = item.message
|
|
conv = item.conversation
|
|
mid = message_id(msg)
|
|
cf = status_map.get(mid, {})
|
|
status = classify(cf)
|
|
|
|
if status == "OK" and not args.include_ok:
|
|
continue
|
|
|
|
post_status = ""
|
|
if args.post_missing and status == "MISSING_IN_CLIENTFLOW":
|
|
result = post_to_clientflow(build_webhook_payload(item))
|
|
if result["ok"]:
|
|
post_status = f"POSTED {result['status']}"
|
|
posted += 1
|
|
else:
|
|
post_status = f"FAILED {result['status']}: {clean(result['raw'], 300)}"
|
|
failed += 1
|
|
|
|
counts[status] = counts.get(status, 0) + 1
|
|
sender = sender_from_conversation(conv, msg)
|
|
rows.append({
|
|
"status": status,
|
|
"chatwoot_created_at": dt_to_display(msg.get("created_at")),
|
|
"conversation_id": str(conv.get("id") or msg.get("conversation_id") or ""),
|
|
"source_event_id": mid,
|
|
"email": str(sender.get("email") or ""),
|
|
"content": clean(message_content(msg)),
|
|
"raw_event_id": cf.get("raw_event_id") or "",
|
|
"processed": cf.get("processed") if cf.get("raw_event_id") else "",
|
|
"ignored": cf.get("ignored") if cf.get("raw_event_id") else "",
|
|
"processing_error": cf.get("processing_error") or "",
|
|
"message_id": cf.get("message_id") or cf.get("raw_event_message_id") or "",
|
|
"communication_id": cf.get("communication_id") or "",
|
|
"opportunity_id": cf.get("opportunity_id") or "",
|
|
"post_status": post_status,
|
|
})
|
|
|
|
write_reports(rows, counts, hours=int(args.hours))
|
|
|
|
print("Auditoria remota Chatwoot inbound concluída.")
|
|
print(f"Mensagens inbound públicas lidas do Chatwoot: {len(remote_messages)}")
|
|
print(f"CSV: {OUT_CSV}")
|
|
print(f"Markdown: {OUT_MD}")
|
|
print("Resumo:")
|
|
if not counts:
|
|
print("OK: sem gaps inbound encontrados")
|
|
for status, total in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
|
|
print(f"{status}: {total}")
|
|
if args.post_missing:
|
|
print(f"posted={posted}")
|
|
print(f"failed={failed}")
|
|
return 0 if failed == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|