Release v4928.1.4.2 stable

scripts/cleanup_invalid_email_identity_suggestions.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Clean stale invalid email-identity suggestions/extractions.
+
+Use after strengthening company-mention filters.  It targets suggestions and
+stored extractions created from invalid fragments such as ``pt`` or ``com``.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Any
+
+from sqlalchemy import text
+
+from app.db import engine
+from app.email_identity_extraction_service import is_plausible_company_mention
+
+INVALID = {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"}
+
+
+def _clean(value: Any) -> str:
+    return str(value or "").strip()
+
+
+def _valid_companies(values: Any) -> list[str]:
+    out: list[str] = []
+    if not isinstance(values, list):
+        return out
+    for value in values:
+        v = _clean(value).strip(" ,.;:-")
+        if v and is_plausible_company_mention(v):
+            out.append(v)
+    return out
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--opportunity-id", default="")
+    parser.add_argument("--apply", action="store_true")
+    parser.add_argument("--include-accepted", action="store_true")
+    parser.add_argument("--fix-extractions", action="store_true", help="Also rewrite stored email_identity_extractions company_mentions after current filters")
+    args = parser.parse_args()
+
+    where = ["lookup_type LIKE 'email_identity%'"]
+    params: dict[str, Any] = {}
+    if args.opportunity_id:
+        where.append("opportunity_id = CAST(:opportunity_id AS UUID)")
+        params["opportunity_id"] = args.opportunity_id
+    if args.include_accepted:
+        where.append("status IN ('pending', 'accepted', 'rejected')")
+    else:
+        where.append("status = 'pending'")
+    invalid_sql = ", ".join("'" + v.replace("'", "") + "'" for v in sorted(INVALID))
+    where.append(f"lower(trim(COALESCE(lookup_value, ''))) IN ({invalid_sql})")
+
+    sql_where = " AND ".join(where)
+    with engine.begin() as conn:
+        rows = conn.execute(text(f"""
+            SELECT id::text, opportunity_id::text, suggested_name, suggested_nif,
+                   lookup_type, lookup_value, confidence, status, reason, created_at
+            FROM fiscal_customer_suggestions
+            WHERE {sql_where}
+            ORDER BY created_at DESC
+        """), params).mappings().all()
+
+        print(json.dumps({"invalid_suggestions": len(rows), "apply": args.apply}, ensure_ascii=False, indent=2, default=str))
+        for row in rows:
+            print(json.dumps(dict(row), ensure_ascii=False, indent=2, default=str))
+
+        if args.apply and rows:
+            ids = [r["id"] for r in rows]
+            conn.execute(text("""
+                UPDATE fiscal_customer_suggestions
+                SET status = 'rejected',
+                    reason = COALESCE(reason, '') || ' | rejected_invalid_email_identity_token',
+                    resolved_by = 'cleanup_invalid_email_identity_suggestions',
+                    resolved_at = now(),
+                    updated_at = now()
+                WHERE id = ANY(CAST(:ids AS UUID[]))
+            """), {"ids": ids})
+            print(json.dumps({"rejected": len(ids)}, ensure_ascii=False, indent=2))
+
+        if args.fix_extractions:
+            extraction_where = []
+            extraction_params: dict[str, Any] = {}
+            if args.opportunity_id:
+                extraction_where.append("opportunity_id = CAST(:opportunity_id AS UUID)")
+                extraction_params["opportunity_id"] = args.opportunity_id
+            extraction_sql = "WHERE " + " AND ".join(extraction_where) if extraction_where else ""
+            ex_rows = conn.execute(text(f"""
+                SELECT id::text, opportunity_id::text, company_mentions, confidence, raw_payload
+                FROM email_identity_extractions
+                {extraction_sql}
+                ORDER BY updated_at DESC
+            """), extraction_params).mappings().all()
+            changed = []
+            for row in ex_rows:
+                original = row.get("company_mentions") or []
+                valid = _valid_companies(original)
+                if list(original or []) == valid:
+                    continue
+                changed.append({"id": row["id"], "opportunity_id": row["opportunity_id"], "before": original, "after": valid})
+                if args.apply:
+                    try:
+                        confidence = float(row.get("confidence") or 0)
+                    except Exception:
+                        confidence = 0.0
+                    if not valid:
+                        confidence = min(confidence, 0.45)
+                    raw_payload = dict(row.get("raw_payload") or {}) if isinstance(row.get("raw_payload"), dict) else {}
+                    raw_payload["filtered_invalid_company_mentions"] = list(original or [])
+                    conn.execute(text("""
+                        UPDATE email_identity_extractions
+                        SET company_mentions = CAST(:company_mentions AS JSONB),
+                            confidence = :confidence,
+                            raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_payload AS JSONB),
+                            updated_at = now()
+                        WHERE id = CAST(:id AS UUID)
+                    """), {
+                        "id": row["id"],
+                        "company_mentions": json.dumps(valid, ensure_ascii=False),
+                        "confidence": confidence,
+                        "raw_payload": json.dumps(raw_payload, ensure_ascii=False, default=str),
+                    })
+            print(json.dumps({"extractions_to_fix": len(changed), "items": changed}, ensure_ascii=False, indent=2, default=str))
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())