Implement document reconciliation v2

app/document_reconciliation_backfill.py

@@ -0,0 +1,119 @@
+"""Pure classification and action building for Document Reconciliation v2 backfills."""
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from typing import Any, Dict, Iterable, List
+
+
+def select_complete_group_batch(rows: Iterable[Dict[str, Any]], batch_size: int,
+                                resume_from: tuple[str, str] | None = None) -> List[Dict[str, Any]]:
+    """Reference group paginator used by tests and non-SQL callers."""
+    ordered = sorted(rows, key=lambda row: (str(row["opportunity_id"]),
+        str(row["document_kind"]), str(row.get("created_at") or ""), str(row["id"])))
+    keys = []
+    for row in ordered:
+        key = (str(row["opportunity_id"]), str(row["document_kind"]))
+        if resume_from and key <= resume_from:
+            continue
+        if key not in keys:
+            if len(keys) >= max(1, int(batch_size)):
+                continue
+            keys.append(key)
+    selected = set(keys)
+    return [row for row in ordered if (str(row["opportunity_id"]), str(row["document_kind"])) in selected]
+
+
+def _document_identity(row: Dict[str, Any]) -> str:
+    return str(row.get("external_id") or row.get("document_number") or row["id"])
+
+
+def _is_cancelled(row: Dict[str, Any]) -> bool:
+    return str(row.get("status") or "").upper() in {"CANCELLED", "CANCELED"}
+
+
+def classify_group(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Classify documents from one opportunity and document-kind group."""
+    active_primaries = [
+        row
+        for row in rows
+        if bool(row.get("is_primary"))
+        and str(row.get("role") or "").lower() in {"current", "accepted"}
+    ]
+    duplicate_identity = len({_document_identity(row) for row in rows}) < len(rows)
+    ambiguous = len(active_primaries) > 1 or duplicate_identity
+    actions = []
+    for row in rows:
+        role = str(row.get("role") or "").lower()
+        evidence = row.get("payload") if isinstance(row.get("payload"), dict) else {}
+        if ambiguous:
+            relationship, reason = "REVIEW_REQUIRED", "multiple primaries or duplicate association"
+        elif _is_cancelled(row) and row in active_primaries:
+            relationship, reason = "REVIEW_REQUIRED", "cancelled document was inferred primary"
+        elif len(active_primaries) == 1 and row["id"] == active_primaries[0]["id"]:
+            relationship, reason = "PRIMARY", "single unambiguous legacy primary"
+        elif role == "historical":
+            relationship, reason = "HISTORICAL", "explicit legacy historical role"
+        elif role in {"related", "accepted"}:
+            relationship, reason = "SECONDARY", "explicit non-primary related/accepted role"
+        elif role == "detached" and any(
+            evidence.get(key)
+            for key in ("manual_unlinked_from_opportunity_id", "removed_reason", "detached_evidence")
+        ):
+            relationship, reason = "REMOVED", "explicit legacy detach evidence"
+        else:
+            relationship, reason = "REVIEW_REQUIRED", "legacy evidence is not unambiguous"
+        actions.append(
+            {
+                **row,
+                "relationship": relationship,
+                "decision_reason": reason,
+                "cancelled": _is_cancelled(row),
+            }
+        )
+    return actions
+
+
+def build_backfill_result(
+    rows: Iterable[Dict[str, Any]],
+    *,
+    lines_without_document: int = 0,
+    documents_in_multiple_opportunities: int = 0,
+) -> Dict[str, Any]:
+    """Build backfill actions and summary from already-loaded document rows."""
+    groups: Dict[tuple, List[Dict[str, Any]]] = defaultdict(list)
+    for row in rows:
+        groups[(row["opportunity_id"], row["document_kind"])].append(row)
+
+    actions = [action for group in groups.values() for action in classify_group(group)]
+    counts = Counter(action["relationship"] for action in actions)
+    current_total = sum(
+        float(action.get("total_amount") or action.get("amount") or 0) for action in actions
+    )
+    primary_total = sum(
+        float(action.get("total_amount") or action.get("amount") or 0)
+        for action in actions
+        if action["relationship"] == "PRIMARY"
+    )
+    duplicates = sum(
+        1
+        for group in groups.values()
+        if len({_document_identity(row) for row in group}) < len(group)
+    )
+    return {
+        "summary": {
+            "total_opportunities": len({action["opportunity_id"] for action in actions}),
+            "total_documents": len(actions),
+            "auto_PRIMARY": counts["PRIMARY"],
+            "auto_SECONDARY": counts["SECONDARY"],
+            "auto_HISTORICAL": counts["HISTORICAL"],
+            "REVIEW_REQUIRED": counts["REVIEW_REQUIRED"],
+            "cancelled": sum(1 for action in actions if action["cancelled"]),
+            "duplicates": duplicates,
+            "conflicts": counts["REVIEW_REQUIRED"],
+            "lines_without_document": lines_without_document,
+            "documents_in_multiple_opportunities": documents_in_multiple_opportunities,
+            "current_total": current_total,
+            "primary_total": primary_total,
+        },
+        "actions": actions,
+    }