118 lines
5.2 KiB
Python
118 lines
5.2 KiB
Python
"""Cleanup helpers for stale email identity suggestions/extractions."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any, Dict, List
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import engine
|
|
from app.email_identity_extraction_service import is_plausible_company_mention
|
|
|
|
INVALID_EMAIL_IDENTITY_TOKENS = {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"}
|
|
|
|
|
|
def _clean(value: Any) -> str:
|
|
return str(value or "").strip()
|
|
|
|
|
|
def _valid_company_mentions(values: Any) -> List[str]:
|
|
out: List[str] = []
|
|
if not isinstance(values, list):
|
|
return out
|
|
for value in values:
|
|
v = _clean(value).strip(" ,.;:-")
|
|
if v and is_plausible_company_mention(v):
|
|
out.append(v)
|
|
return out
|
|
|
|
|
|
def cleanup_invalid_email_identity_state(
|
|
*,
|
|
opportunity_id: str | None = None,
|
|
include_accepted: bool = True,
|
|
fix_extractions: bool = True,
|
|
apply: bool = True,
|
|
) -> Dict[str, Any]:
|
|
where = ["lookup_type LIKE 'email_identity%'"]
|
|
params: dict[str, Any] = {}
|
|
if opportunity_id:
|
|
where.append("opportunity_id = CAST(:opportunity_id AS UUID)")
|
|
params["opportunity_id"] = opportunity_id
|
|
where.append("status IN ('pending', 'accepted', 'rejected')" if include_accepted else "status = 'pending'")
|
|
invalid_sql = ", ".join("'" + v.replace("'", "") + "'" for v in sorted(INVALID_EMAIL_IDENTITY_TOKENS))
|
|
where.append(f"lower(trim(COALESCE(lookup_value, ''))) IN ({invalid_sql})")
|
|
sql_where = " AND ".join(where)
|
|
|
|
result: Dict[str, Any] = {"invalid_suggestions": 0, "rejected": 0, "extractions_to_fix": 0, "fixed_extractions": 0}
|
|
with engine.begin() as conn:
|
|
rows = conn.execute(text(f"""
|
|
SELECT id::text, opportunity_id::text, suggested_name, suggested_nif,
|
|
lookup_type, lookup_value, confidence, status, reason, created_at
|
|
FROM fiscal_customer_suggestions
|
|
WHERE {sql_where}
|
|
ORDER BY created_at DESC
|
|
"""), params).mappings().all()
|
|
result["invalid_suggestions"] = len(rows)
|
|
result["suggestions"] = [dict(row) for row in rows]
|
|
if apply and rows:
|
|
ids = [r["id"] for r in rows]
|
|
conn.execute(text("""
|
|
UPDATE fiscal_customer_suggestions
|
|
SET status = 'rejected',
|
|
reason = COALESCE(reason, '') || ' | rejected_invalid_email_identity_token',
|
|
resolved_by = 'cleanup_invalid_email_identity_state',
|
|
resolved_at = now(),
|
|
updated_at = now()
|
|
WHERE id = ANY(CAST(:ids AS UUID[]))
|
|
"""), {"ids": ids})
|
|
result["rejected"] = len(ids)
|
|
|
|
if fix_extractions:
|
|
extraction_where = []
|
|
extraction_params: dict[str, Any] = {}
|
|
if opportunity_id:
|
|
extraction_where.append("opportunity_id = CAST(:opportunity_id AS UUID)")
|
|
extraction_params["opportunity_id"] = opportunity_id
|
|
extraction_sql = "WHERE " + " AND ".join(extraction_where) if extraction_where else ""
|
|
ex_rows = conn.execute(text(f"""
|
|
SELECT id::text, opportunity_id::text, company_mentions, confidence, raw_payload
|
|
FROM email_identity_extractions
|
|
{extraction_sql}
|
|
ORDER BY updated_at DESC
|
|
"""), extraction_params).mappings().all()
|
|
changed = []
|
|
for row in ex_rows:
|
|
original = row.get("company_mentions") or []
|
|
valid = _valid_company_mentions(original)
|
|
if list(original or []) == valid:
|
|
continue
|
|
item = {"id": row["id"], "opportunity_id": row["opportunity_id"], "before": original, "after": valid}
|
|
changed.append(item)
|
|
if apply:
|
|
try:
|
|
confidence = float(row.get("confidence") or 0)
|
|
except Exception:
|
|
confidence = 0.0
|
|
if not valid:
|
|
confidence = min(confidence, 0.45)
|
|
raw_payload = dict(row.get("raw_payload") or {}) if isinstance(row.get("raw_payload"), dict) else {}
|
|
raw_payload["filtered_invalid_company_mentions"] = list(original or [])
|
|
conn.execute(text("""
|
|
UPDATE email_identity_extractions
|
|
SET company_mentions = CAST(:company_mentions AS JSONB),
|
|
confidence = :confidence,
|
|
raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_payload AS JSONB),
|
|
updated_at = now()
|
|
WHERE id = CAST(:id AS UUID)
|
|
"""), {
|
|
"id": row["id"],
|
|
"company_mentions": json.dumps(valid, ensure_ascii=False),
|
|
"confidence": confidence,
|
|
"raw_payload": json.dumps(raw_payload, ensure_ascii=False, default=str),
|
|
})
|
|
result["extractions_to_fix"] = len(changed)
|
|
result["fixed_extractions"] = len(changed) if apply else 0
|
|
result["extractions"] = changed
|
|
return result
|