"""Cleanup helpers for stale email identity suggestions/extractions.""" from __future__ import annotations import json from typing import Any, Dict, List from sqlalchemy import text from app.db import engine from app.email_identity_extraction_service import is_plausible_company_mention INVALID_EMAIL_IDENTITY_TOKENS = {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"} def _clean(value: Any) -> str: return str(value or "").strip() def _valid_company_mentions(values: Any) -> List[str]: out: List[str] = [] if not isinstance(values, list): return out for value in values: v = _clean(value).strip(" ,.;:-") if v and is_plausible_company_mention(v): out.append(v) return out def cleanup_invalid_email_identity_state( *, opportunity_id: str | None = None, include_accepted: bool = True, fix_extractions: bool = True, apply: bool = True, ) -> Dict[str, Any]: where = ["lookup_type LIKE 'email_identity%'"] params: dict[str, Any] = {} if opportunity_id: where.append("opportunity_id = CAST(:opportunity_id AS UUID)") params["opportunity_id"] = opportunity_id where.append("status IN ('pending', 'accepted', 'rejected')" if include_accepted else "status = 'pending'") invalid_sql = ", ".join("'" + v.replace("'", "") + "'" for v in sorted(INVALID_EMAIL_IDENTITY_TOKENS)) where.append(f"lower(trim(COALESCE(lookup_value, ''))) IN ({invalid_sql})") sql_where = " AND ".join(where) result: Dict[str, Any] = {"invalid_suggestions": 0, "rejected": 0, "extractions_to_fix": 0, "fixed_extractions": 0} with engine.begin() as conn: rows = conn.execute(text(f""" SELECT id::text, opportunity_id::text, suggested_name, suggested_nif, lookup_type, lookup_value, confidence, status, reason, created_at FROM fiscal_customer_suggestions WHERE {sql_where} ORDER BY created_at DESC """), params).mappings().all() result["invalid_suggestions"] = len(rows) result["suggestions"] = [dict(row) for row in rows] if apply and rows: ids = [r["id"] for r in rows] conn.execute(text(""" UPDATE fiscal_customer_suggestions SET status = 'rejected', reason = COALESCE(reason, '') || ' | rejected_invalid_email_identity_token', resolved_by = 'cleanup_invalid_email_identity_state', resolved_at = now(), updated_at = now() WHERE id = ANY(CAST(:ids AS UUID[])) """), {"ids": ids}) result["rejected"] = len(ids) if fix_extractions: extraction_where = [] extraction_params: dict[str, Any] = {} if opportunity_id: extraction_where.append("opportunity_id = CAST(:opportunity_id AS UUID)") extraction_params["opportunity_id"] = opportunity_id extraction_sql = "WHERE " + " AND ".join(extraction_where) if extraction_where else "" ex_rows = conn.execute(text(f""" SELECT id::text, opportunity_id::text, company_mentions, confidence, raw_payload FROM email_identity_extractions {extraction_sql} ORDER BY updated_at DESC """), extraction_params).mappings().all() changed = [] for row in ex_rows: original = row.get("company_mentions") or [] valid = _valid_company_mentions(original) if list(original or []) == valid: continue item = {"id": row["id"], "opportunity_id": row["opportunity_id"], "before": original, "after": valid} changed.append(item) if apply: try: confidence = float(row.get("confidence") or 0) except Exception: confidence = 0.0 if not valid: confidence = min(confidence, 0.45) raw_payload = dict(row.get("raw_payload") or {}) if isinstance(row.get("raw_payload"), dict) else {} raw_payload["filtered_invalid_company_mentions"] = list(original or []) conn.execute(text(""" UPDATE email_identity_extractions SET company_mentions = CAST(:company_mentions AS JSONB), confidence = :confidence, raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_payload AS JSONB), updated_at = now() WHERE id = CAST(:id AS UUID) """), { "id": row["id"], "company_mentions": json.dumps(valid, ensure_ascii=False), "confidence": confidence, "raw_payload": json.dumps(raw_payload, ensure_ascii=False, default=str), }) result["extractions_to_fix"] = len(changed) result["fixed_extractions"] = len(changed) if apply else 0 result["extractions"] = changed return result