fix: canonicalize operations work items

app/operations_service.py

@@ -21,6 +21,13 @@ from app.work_center_action_policy import (
     canonical_action_code,
     reconstructed_review_required,
 )
+from app.canonical_operations import (
+    canonicalize_operations,
+    opportunity_ids_from_work_seeds,
+    partition_canonical_items,
+)
+from app.opportunity_next_action_service import get_opportunity_next_actions
+from app.document_reconciliation_service import active_document_link_exclusion_sql
 
 
 def _int(value: Any) -> int:
@@ -303,6 +310,11 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
     Technical lists remain available in their own pages and should only appear
     here when they block an operator action.
     """
+    display_limit = max(1, min(int(limit), 200))
+    # Bound each entity source independently, then apply the user-facing limit
+    # after canonicalization. This prevents duplicate rows from consuming the
+    # display limit while still avoiding an unbounded history scan.
+    candidate_limit = max(200, min(display_limit * 20, 1000))
     with engine.begin() as conn:
         counts = conn.execute(text("""
             SELECT
@@ -383,7 +395,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
             LIMIT :limit
         """), {"limit": int(limit)}).mappings().all()
 
-        work_items = conn.execute(text("""
+        work_seed_rows = conn.execute(text("""
             SELECT * FROM (
                 SELECT 'task' AS source, t.id::text AS id, t.created_at, t.due_at,
                        CASE
@@ -433,14 +445,19 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
                        COALESCE(o.stage, '') AS opportunity_stage,
                        COALESCE(o.value_amount, 0) AS opportunity_value_amount,
                        COALESCE(o.currency, 'EUR') AS opportunity_currency
-                FROM tasks t
+                FROM (
+                    SELECT * FROM tasks
+                    WHERE status = 'pending'
+                      AND NOT (action_code LIKE 'FOLLOW_UP_%' AND due_at IS NOT NULL AND due_at > now())
+                    ORDER BY created_at DESC
+                    LIMIT :candidate_limit
+                ) t
                 LEFT JOIN opportunities o ON o.id = t.opportunity_id
                 LEFT JOIN customers cu_opp ON cu_opp.id = o.local_customer_id
                 LEFT JOIN customers cu_task ON cu_task.id::text = t.customer_id
                 LEFT JOIN messages m ON m.id = t.message_id
                 LEFT JOIN raw_events re ON re.id = t.raw_event_id
-                WHERE t.status = 'pending'
-                  AND NOT (t.action_code LIKE 'FOLLOW_UP_%' AND t.due_at IS NOT NULL AND t.due_at > now())
+                WHERE TRUE
 
                 UNION ALL
 
@@ -477,11 +494,16 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
                        COALESCE(o.stage, '') AS opportunity_stage,
                        COALESCE(o.value_amount, 0) AS opportunity_value_amount,
                        COALESCE(o.currency, 'EUR') AS opportunity_currency
-                FROM integration_outbox io
+                FROM (
+                    SELECT * FROM integration_outbox
+                    WHERE status IN ('failed','blocked')
+                      AND NOT (status = 'failed' AND (COALESCE(last_error,'') ILIKE '%limpo manualmente%' OR COALESCE(last_error,'') ILIKE '%resolvido manualmente%'))
+                    ORDER BY created_at DESC
+                    LIMIT :candidate_limit
+                ) io
                 LEFT JOIN opportunities o ON o.id::text = NULLIF(io.payload->>'opportunity_id','')
                 LEFT JOIN customers cu ON cu.id = o.local_customer_id
-                WHERE io.status IN ('pending','failed','blocked')
-                  AND NOT (io.status = 'failed' AND (COALESCE(io.last_error,'') ILIKE '%limpo manualmente%' OR COALESCE(io.last_error,'') ILIKE '%resolvido manualmente%'))
+                WHERE TRUE
 
                 UNION ALL
 
@@ -519,29 +541,139 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
                        '/communications/' || c.id::text AS href,
                        CASE WHEN c.customer_id IS NULL THEN 'Associar cliente' ELSE 'Abrir' END AS action_label,
                        ''::text AS opportunity_linking_status,
-                       COALESCE(c.metadata, '{}'::jsonb) AS item_metadata,
+                       COALESCE(c.metadata, '{}'::jsonb)
+                         || CASE WHEN c.task_id IS NOT NULL
+                                 THEN jsonb_build_object('task_id', c.task_id::text)
+                                 ELSE '{}'::jsonb END AS item_metadata,
                        COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
                        COALESCE(o.stage, '') AS opportunity_stage,
                        COALESCE(o.value_amount, 0) AS opportunity_value_amount,
                        COALESCE(o.currency, 'EUR') AS opportunity_currency
-                FROM communications c
+                FROM (
+                    SELECT * FROM communications
+                    WHERE status IN ('new','classified','needs_review')
+                    ORDER BY created_at DESC
+                    LIMIT :candidate_limit
+                ) c
                 LEFT JOIN customers cu ON cu.id = c.customer_id
                 LEFT JOIN opportunities o ON o.id = c.opportunity_id
-                WHERE c.status IN ('new','classified','needs_review')
+                WHERE TRUE
+
+                UNION ALL
+
+                SELECT 'reconciliation' AS source, ri.id::text AS id, ri.created_at, NULL::timestamptz AS due_at,
+                       COALESCE(ri.priority, 'normal') AS priority,
+                       'rever' AS queue,
+                       upper(COALESCE(ri.suggested_action, 'RECONCILE_DOCUMENTS')) AS action_code,
+                       COALESCE(ri.title, 'Rever reconciliação') AS title,
+                       COALESCE(ri.description, ri.resolution_note, '') AS detail,
+                       ri.status,
+                       ri.source_system,
+                       NULL::text AS conversation_id,
+                       NULL::text AS contact_id,
+                       COALESCE(ri.description, '') AS request_text,
+                       ri.opportunity_id::text,
+                       COALESCE(o.title, '') AS opportunity_title,
+                       COALESCE(cu.name, ri.customer_name, ri.customer_email, '') AS customer_name,
+                       COALESCE(ri.customer_name, '') AS sender_name,
+                       COALESCE(ri.customer_email, '') AS sender_email,
+                       COALESCE(cu.name, '') AS fiscal_customer_name,
+                       COALESCE(cu.email, '') AS fiscal_customer_email,
+                       COALESCE(cu.tax_id, '') AS fiscal_customer_tax_id,
+                       COALESCE(cu.street_name, '') AS fiscal_customer_street_name,
+                       COALESCE(cu.postal_zone, '') AS fiscal_customer_postal_zone,
+                       COALESCE(cu.city_name, '') AS fiscal_customer_city_name,
+                       COALESCE(cu.name, ri.customer_name, ri.customer_email, '') AS contact_display_name,
+                       COALESCE(ri.customer_email, '') AS customer_email,
+                       ''::text AS no_opportunity_reason,
+                       '/reconciliation' AS href,
+                       'Rever' AS action_label,
+                       ''::text AS opportunity_linking_status,
+                       COALESCE(ri.payload, '{}'::jsonb) AS item_metadata,
+                       COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
+                       COALESCE(o.stage, '') AS opportunity_stage,
+                       COALESCE(o.value_amount, ri.amount, 0) AS opportunity_value_amount,
+                       COALESCE(o.currency, ri.currency, 'EUR') AS opportunity_currency
+                FROM (
+                    SELECT * FROM reconciliation_items
+                    WHERE status IN ('open','needs_review','conflict')
+                      AND """ + active_document_link_exclusion_sql("reconciliation_items") + """
+                    ORDER BY created_at DESC
+                    LIMIT :candidate_limit
+                ) ri
+                LEFT JOIN opportunities o ON o.id = ri.opportunity_id
+                LEFT JOIN customers cu ON cu.id = COALESCE(o.local_customer_id, ri.customer_id)
+                WHERE TRUE
             ) items
             ORDER BY
               CASE lower(priority) WHEN 'alta' THEN 1 WHEN 'high' THEN 1 WHEN 'urgente' THEN 0 WHEN 'normal' THEN 2 ELSE 3 END,
               created_at DESC
-            LIMIT :limit
-        """), {"limit": int(limit)}).mappings().all()
+        """), {"candidate_limit": candidate_limit}).mappings().all()
 
-    cleaned_work_items = _attach_operation_urls(_normalise_work_item_intent([dict(r) for r in work_items]))
+        seed_opportunity_ids = opportunity_ids_from_work_seeds(work_seed_rows)
+        evidence_rows = []
+        if seed_opportunity_ids:
+            evidence_rows = conn.execute(text("""
+                SELECT * FROM (
+                    SELECT 'communication' AS source, c.id::text AS id, c.created_at,
+                           c.status, upper(COALESCE(c.classification, 'REVIEW_MANUALLY')) AS action_code,
+                           c.opportunity_id::text, c.source_system, c.conversation_id,
+                           COALESCE(c.metadata, '{}'::jsonb)
+                             || CASE WHEN c.task_id IS NOT NULL
+                                     THEN jsonb_build_object('task_id', c.task_id::text)
+                                     ELSE '{}'::jsonb END AS item_metadata
+                    FROM communications c
+                    WHERE c.opportunity_id = ANY(CAST(:opportunity_ids AS UUID[]))
+                      AND c.status NOT IN ('new','classified','needs_review')
+                    ORDER BY c.created_at DESC
+                    LIMIT :candidate_limit
+                ) communication_evidence
+                UNION ALL
+                SELECT * FROM (
+                    SELECT 'reconciliation' AS source, ri.id::text AS id, ri.created_at,
+                           ri.status, upper(COALESCE(ri.suggested_action, 'RECONCILE_DOCUMENTS')) AS action_code,
+                           ri.opportunity_id::text, ri.source_system, NULL::text AS conversation_id,
+                           COALESCE(ri.payload, '{}'::jsonb) AS item_metadata
+                    FROM reconciliation_items ri
+                    WHERE ri.opportunity_id = ANY(CAST(:opportunity_ids AS UUID[]))
+                      AND ri.status IN ('linked','resolved')
+                    ORDER BY ri.created_at DESC
+                    LIMIT :candidate_limit
+                ) reconciliation_evidence
+            """), {
+                "opportunity_ids": seed_opportunity_ids,
+                "candidate_limit": candidate_limit,
+            }).mappings().all()
+
+    normalized_rows = _normalise_work_item_intent([dict(r) for r in work_seed_rows])
+    opportunity_ids = opportunity_ids_from_work_seeds(normalized_rows)
+    decisions = get_opportunity_next_actions(opportunity_ids) if opportunity_ids else {}
+    projection = canonicalize_operations(normalized_rows, decisions, evidence_rows=[dict(row) for row in evidence_rows])
+    canonical_items = _attach_operation_urls(list(projection["items"]))
+    partition = partition_canonical_items(canonical_items, display_limit=display_limit)
+    actionable_items = partition["actionable_items"]
+    all_waiting_items = partition["waiting_items"]
+    cleaned_work_items = partition["visible_actionable_items"]
+    waiting_items = partition["visible_waiting_items"]
     cleaned_counts = {k: _int(v) for k, v in dict(counts).items()}
     # v4.9.0: the visible Operations total should match the queue the
     # operator can actually act on, not raw pending tasks that include mailbox
     # noise awaiting cleanup. The cleanup script still fixes the data source.
-    cleaned_counts["work_queue_total"] = len(cleaned_work_items)
+    cleaned_counts["work_queue_total"] = partition["work_queue_total"]
+    cleaned_counts["waiting_total"] = partition["waiting_total"]
+    cleaned_counts["raw_source_count"] = projection["raw_source_count"]
+    cleaned_counts["canonical_total"] = len(canonical_items)
 
+    diagnostics = {
+        "candidate_limit": candidate_limit,
+        "raw_source_count": projection["raw_source_count"],
+        "clean_source_count": projection["clean_source_count"],
+        "seed_source_count": projection["seed_source_count"],
+        "evidence_source_count": projection["evidence_source_count"],
+        "canonical_count": len(canonical_items),
+        "visible_count": len(cleaned_work_items),
+        "waiting_total": len(all_waiting_items),
+    }
     return {
         "counts": cleaned_counts,
         "recent_outbox": [dict(r) for r in recent_outbox],
@@ -550,6 +682,9 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
         "incomplete_customers": [dict(r) for r in incomplete_customers],
         "recent_communications": [dict(r) for r in recent_communications],
         "work_items": cleaned_work_items,
+        "waiting_items": waiting_items,
+        **diagnostics,
+        "projection_metrics": diagnostics,
     }