clientflow_backend/scripts/evaluate_action_decider.py

#!/usr/bin/env python3
import json
import os
import sys
import urllib.request
import urllib.error
from pathlib import Path
from typing import Any, Dict, List

try:
    import psycopg
except Exception:
    psycopg = None


def env(name: str, default: str = "") -> str:
    return os.getenv(name, default).strip()


API_URL = env("EVAL_API_URL", "http://127.0.0.1:8020/analyze")
DATASET = Path(env("EVAL_DATASET", "data/action_eval_cases.jsonl"))
PSQL_DATABASE_URL = env("PSQL_DATABASE_URL")
EVAL_CLEANUP = env("EVAL_CLEANUP", "true").lower() == "true"
EVAL_LIMIT = int(env("EVAL_LIMIT", "0") or "0")
EVAL_LLM_ONLY = env("EVAL_LLM_ONLY", "false").lower() == "true"


def load_cases() -> List[Dict[str, Any]]:
    cases = []
    with DATASET.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                cases.append(json.loads(line))

    if EVAL_LIMIT > 0:
        return cases[:EVAL_LIMIT]

    return cases


def post_json(url: str, payload: Dict[str, Any]) -> Dict[str, Any]:
    raw = json.dumps(payload, ensure_ascii=False).encode("utf-8")

    req = urllib.request.Request(
        url,
        data=raw,
        headers={"Content-Type": "application/json"},
        method="POST",
    )

    try:
        with urllib.request.urlopen(req, timeout=90) as resp:
            body = resp.read().decode("utf-8", errors="replace")
            return {
                "ok": 200 <= resp.status < 300,
                "status": resp.status,
                "json": json.loads(body) if body else {},
                "raw": body,
            }
    except urllib.error.HTTPError as e:
        body = e.read().decode("utf-8", errors="replace")
        return {
            "ok": False,
            "status": e.code,
            "json": None,
            "raw": body,
        }


def cleanup_eval_data() -> None:
    if not EVAL_CLEANUP or not PSQL_DATABASE_URL or psycopg is None:
        return

    # Limpeza defensiva: algumas tabelas não têm source_system/source_event_id.
    # Por isso verificamos as colunas reais antes de construir o DELETE.
    tables = [
        "integration_outbox",
        "tasks",
        "action_runs",
        "messages",
        "raw_events",
    ]

    def table_columns(cur, table: str) -> set[str]:
        cur.execute(
            """
            select column_name
            from information_schema.columns
            where table_name = %s
            """,
            (table,),
        )
        return {row[0] for row in cur.fetchall()}

    with psycopg.connect(PSQL_DATABASE_URL) as conn:
        with conn.cursor() as cur:
            for table in tables:
                cols = table_columns(cur, table)
                conditions = []

                if "source_system" in cols:
                    conditions.append("source_system like 'action_eval%'")

                if "conversation_id" in cols:
                    conditions.append("conversation_id like 'eval-%'")

                if "contact_id" in cols:
                    conditions.append("contact_id like 'eval-contact-%'")

                if "source_event_id" in cols:
                    conditions.append("source_event_id like 'eval-%'")

                if "idempotency_key" in cols:
                    conditions.append("idempotency_key like '%eval-%'")

                if "metadata" in cols:
                    conditions.append("metadata::text like '%action_eval%'")

                if not conditions:
                    continue

                sql = f"delete from {table} where " + " or ".join(conditions)
                cur.execute(sql)

        conn.commit()


def main() -> int:
    cases = load_cases()

    cleanup_eval_data()

    results = []
    ok_count = 0

    print(f"Dataset: {DATASET}")
    print(f"Cases: {len(cases)}")
    print(f"API: {API_URL}")
    print(f"EVAL_LLM_ONLY={EVAL_LLM_ONLY}")
    print("---")

    for i, case in enumerate(cases, start=1):
        conv_id = f"eval-{case['id']}"
        payload = {
            "last_customer_message": case["message"],
            "previous_context": case.get("context", ""),
            "source": "action_eval_llm_only" if EVAL_LLM_ONLY else "action_eval",
            "conversation_id": conv_id,
            "contact_id": f"eval-contact-{i}",
        }

        result = post_json(API_URL, payload)

        if not result["ok"]:
            got = "HTTP_ERROR"
            confidence = None
            provider = None
            note = result["raw"][:300]
        else:
            data = result["json"] or {}
            decision = data.get("action_decision") or {}
            usage = data.get("usage") or {}
            got = decision.get("action_code") or data.get("action_result", {}).get("action_code") or "UNKNOWN"
            confidence = decision.get("confidence")
            provider = usage.get("provider")
            note = decision.get("note")

        expected = case["expected"]
        passed = got == expected
        ok_count += 1 if passed else 0

        results.append({
            "id": case["id"],
            "expected": expected,
            "got": got,
            "ok": passed,
            "confidence": confidence,
            "provider": provider,
            "note": note,
        })

        status = "OK" if passed else "FAIL"
        print(f"{status:4} {case['id']}")
        print(f"     expected={expected}")
        print(f"     got     ={got}")
        print(f"     conf    ={confidence} provider={provider}")
        if not passed:
            print(f"     note    ={note}")
        print("---")

    accuracy = ok_count / len(cases) if cases else 0

    print(f"accuracy={accuracy:.1%} ({ok_count}/{len(cases)})")

    by_expected = {}
    for r in results:
        item = by_expected.setdefault(r["expected"], {"ok": 0, "total": 0})
        item["total"] += 1
        item["ok"] += 1 if r["ok"] else 0

    print("--- by expected")
    for code, stats in sorted(by_expected.items()):
        print(f"{code}: {stats['ok']}/{stats['total']}")

    Path("data/action_eval_last_results.json").write_text(
        json.dumps(results, ensure_ascii=False, indent=2),
        encoding="utf-8",
    )

    cleanup_eval_data()

    return 0 if accuracy >= 0.90 else 1


if __name__ == "__main__":
    raise SystemExit(main())