clientflow_backend/scripts/audit_chatwoot_ingestion_gap.py

#!/usr/bin/env python3
"""Audita lacunas Chatwoot -> ClientFlow usando payload.message_type.

A versão anterior gerava falsos positivos porque assumia sender.type=contact;
nos webhooks de email do Chatwoot, inbound chega como message_type=incoming e
sender.type pode estar vazio.
"""
from __future__ import annotations

import argparse
import csv
from pathlib import Path
from typing import Any

from sqlalchemy import text

from app.db import engine

OUT_CSV = Path("/tmp/clientflow_chatwoot_ingestion_gap.csv")
OUT_MD = Path("/tmp/clientflow_chatwoot_ingestion_gap.md")


def clean(value: Any, limit: int = 240) -> str:
    txt = " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split())
    return txt if len(txt) <= limit else txt[: limit - 1].rstrip() + "…"


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--hours", type=int, default=168)
    parser.add_argument("--include-ok", action="store_true")
    args = parser.parse_args()

    with engine.begin() as conn:
        rows = conn.execute(text("""
            SELECT
                re.id::text AS raw_event_id,
                re.source_event_id,
                re.created_at,
                re.processed,
                re.ignored,
                re.processing_error,
                re.payload #>> '{message_type}' AS message_type,
                re.payload #>> '{conversation,id}' AS conversation_id,
                re.payload #>> '{sender,email}' AS sender_email,
                re.payload #>> '{conversation,meta,sender,email}' AS meta_sender_email,
                left(coalesce(re.payload #>> '{content}', ''), 600) AS content,
                m.id IS NOT NULL AS has_message,
                c.id IS NOT NULL AS has_communication,
                o.id IS NOT NULL AS has_opportunity
            FROM raw_events re
            LEFT JOIN messages m
              ON m.source_system = 'chatwoot'
             AND m.source_event_id = re.source_event_id
            LEFT JOIN communications c
              ON c.source_system = 'chatwoot'
             AND c.source_message_id = re.source_event_id
            LEFT JOIN opportunities o
              ON o.conversation_id = re.payload #>> '{conversation,id}'
            WHERE re.source_system = 'chatwoot'
              AND re.event_type = 'message_created'
              AND re.created_at >= now() - (:hours * interval '1 hour')
            ORDER BY re.created_at DESC
        """), {"hours": int(args.hours)}).mappings().all()

    report: list[dict[str, Any]] = []
    counts: dict[str, int] = {}

    for row in rows:
        message_type = str(row.get("message_type") or "").lower()
        if message_type == "outgoing":
            status = "OUTGOING_IGNORED_FOR_INBOUND_AUDIT"
            if not args.include_ok:
                continue
        elif message_type != "incoming":
            status = "NON_INCOMING_OR_UNKNOWN"
            if not args.include_ok:
                continue
        elif not row.get("processed") and not row.get("ignored") and not row.get("processing_error"):
            status = "PENDING_INCOMING_RAW_EVENT"
        elif row.get("processing_error"):
            status = "INCOMING_PROCESSING_ERROR"
        elif row.get("ignored"):
            status = "INCOMING_IGNORED"
        elif not row.get("has_message") and not row.get("has_communication"):
            status = "PROCESSED_BUT_NOT_VISIBLE"
        elif not row.get("has_opportunity"):
            status = "VISIBLE_BUT_UNLINKED_CONVERSATION"
        else:
            status = "OK"
            if not args.include_ok:
                continue

        counts[status] = counts.get(status, 0) + 1
        report.append({
            "status": status,
            "created_at": row.get("created_at"),
            "raw_event_id": row.get("raw_event_id"),
            "source_event_id": row.get("source_event_id"),
            "conversation_id": row.get("conversation_id"),
            "email": row.get("sender_email") or row.get("meta_sender_email") or "",
            "processed": row.get("processed"),
            "ignored": row.get("ignored"),
            "processing_error": row.get("processing_error") or "",
            "has_message": row.get("has_message"),
            "has_communication": row.get("has_communication"),
            "has_opportunity": row.get("has_opportunity"),
            "content": clean(row.get("content")),
        })

    fieldnames = list(report[0].keys()) if report else [
        "status", "created_at", "raw_event_id", "source_event_id", "conversation_id", "email",
        "processed", "ignored", "processing_error", "has_message", "has_communication", "has_opportunity", "content",
    ]
    with OUT_CSV.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(report)

    with OUT_MD.open("w", encoding="utf-8") as f:
        f.write("# Auditoria Chatwoot → ClientFlow\n\n")
        f.write(f"Janela: últimas {args.hours} horas\n\n")
        f.write("## Resumo\n\n")
        for status, total in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
            f.write(f"- {status}: {total}\n")
        f.write("\n## Casos\n\n")
        for row in report[:200]:
            f.write(f"### {row['status']} · conv #{row['conversation_id']} · msg {row['source_event_id']}\n\n")
            f.write(f"- Email: {row['email'] or '-'}\n")
            f.write(f"- Data: {row['created_at']}\n")
            f.write(f"- Erro: {row['processing_error'] or '-'}\n")
            f.write(f"- Excerto: {row['content'] or '-'}\n\n")

    print("Auditoria Chatwoot inbound concluída.")
    print(f"CSV: {OUT_CSV}")
    print(f"Markdown: {OUT_MD}")
    print("Resumo:")
    if not counts:
        print("OK: sem gaps inbound encontrados")
    for status, total in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
        print(f"{status}: {total}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())