132 lines
5.6 KiB
Python
Executable File
132 lines
5.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Clean stale invalid email-identity suggestions/extractions.
|
|
|
|
Use after strengthening company-mention filters. It targets suggestions and
|
|
stored extractions created from invalid fragments such as ``pt`` or ``com``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from typing import Any
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import engine
|
|
from app.email_identity_extraction_service import is_plausible_company_mention
|
|
|
|
INVALID = {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"}
|
|
|
|
|
|
def _clean(value: Any) -> str:
|
|
return str(value or "").strip()
|
|
|
|
|
|
def _valid_companies(values: Any) -> list[str]:
|
|
out: list[str] = []
|
|
if not isinstance(values, list):
|
|
return out
|
|
for value in values:
|
|
v = _clean(value).strip(" ,.;:-")
|
|
if v and is_plausible_company_mention(v):
|
|
out.append(v)
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--opportunity-id", default="")
|
|
parser.add_argument("--apply", action="store_true")
|
|
parser.add_argument("--include-accepted", action="store_true")
|
|
parser.add_argument("--fix-extractions", action="store_true", help="Also rewrite stored email_identity_extractions company_mentions after current filters")
|
|
args = parser.parse_args()
|
|
|
|
where = ["lookup_type LIKE 'email_identity%'"]
|
|
params: dict[str, Any] = {}
|
|
if args.opportunity_id:
|
|
where.append("opportunity_id = CAST(:opportunity_id AS UUID)")
|
|
params["opportunity_id"] = args.opportunity_id
|
|
if args.include_accepted:
|
|
where.append("status IN ('pending', 'accepted', 'rejected')")
|
|
else:
|
|
where.append("status = 'pending'")
|
|
invalid_sql = ", ".join("'" + v.replace("'", "") + "'" for v in sorted(INVALID))
|
|
where.append(f"lower(trim(COALESCE(lookup_value, ''))) IN ({invalid_sql})")
|
|
|
|
sql_where = " AND ".join(where)
|
|
with engine.begin() as conn:
|
|
rows = conn.execute(text(f"""
|
|
SELECT id::text, opportunity_id::text, suggested_name, suggested_nif,
|
|
lookup_type, lookup_value, confidence, status, reason, created_at
|
|
FROM fiscal_customer_suggestions
|
|
WHERE {sql_where}
|
|
ORDER BY created_at DESC
|
|
"""), params).mappings().all()
|
|
|
|
print(json.dumps({"invalid_suggestions": len(rows), "apply": args.apply}, ensure_ascii=False, indent=2, default=str))
|
|
for row in rows:
|
|
print(json.dumps(dict(row), ensure_ascii=False, indent=2, default=str))
|
|
|
|
if args.apply and rows:
|
|
ids = [r["id"] for r in rows]
|
|
conn.execute(text("""
|
|
UPDATE fiscal_customer_suggestions
|
|
SET status = 'rejected',
|
|
reason = COALESCE(reason, '') || ' | rejected_invalid_email_identity_token',
|
|
resolved_by = 'cleanup_invalid_email_identity_suggestions',
|
|
resolved_at = now(),
|
|
updated_at = now()
|
|
WHERE id = ANY(CAST(:ids AS UUID[]))
|
|
"""), {"ids": ids})
|
|
print(json.dumps({"rejected": len(ids)}, ensure_ascii=False, indent=2))
|
|
|
|
if args.fix_extractions:
|
|
extraction_where = []
|
|
extraction_params: dict[str, Any] = {}
|
|
if args.opportunity_id:
|
|
extraction_where.append("opportunity_id = CAST(:opportunity_id AS UUID)")
|
|
extraction_params["opportunity_id"] = args.opportunity_id
|
|
extraction_sql = "WHERE " + " AND ".join(extraction_where) if extraction_where else ""
|
|
ex_rows = conn.execute(text(f"""
|
|
SELECT id::text, opportunity_id::text, company_mentions, confidence, raw_payload
|
|
FROM email_identity_extractions
|
|
{extraction_sql}
|
|
ORDER BY updated_at DESC
|
|
"""), extraction_params).mappings().all()
|
|
changed = []
|
|
for row in ex_rows:
|
|
original = row.get("company_mentions") or []
|
|
valid = _valid_companies(original)
|
|
if list(original or []) == valid:
|
|
continue
|
|
changed.append({"id": row["id"], "opportunity_id": row["opportunity_id"], "before": original, "after": valid})
|
|
if args.apply:
|
|
try:
|
|
confidence = float(row.get("confidence") or 0)
|
|
except Exception:
|
|
confidence = 0.0
|
|
if not valid:
|
|
confidence = min(confidence, 0.45)
|
|
raw_payload = dict(row.get("raw_payload") or {}) if isinstance(row.get("raw_payload"), dict) else {}
|
|
raw_payload["filtered_invalid_company_mentions"] = list(original or [])
|
|
conn.execute(text("""
|
|
UPDATE email_identity_extractions
|
|
SET company_mentions = CAST(:company_mentions AS JSONB),
|
|
confidence = :confidence,
|
|
raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_payload AS JSONB),
|
|
updated_at = now()
|
|
WHERE id = CAST(:id AS UUID)
|
|
"""), {
|
|
"id": row["id"],
|
|
"company_mentions": json.dumps(valid, ensure_ascii=False),
|
|
"confidence": confidence,
|
|
"raw_payload": json.dumps(raw_payload, ensure_ascii=False, default=str),
|
|
})
|
|
print(json.dumps({"extractions_to_fix": len(changed), "items": changed}, ensure_ascii=False, indent=2, default=str))
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|