fix: canonicalize operations work items
This commit is contained in:
plx
474
app/canonical_operations.py
Normal file
474
app/canonical_operations.py
Normal file
@@ -0,0 +1,474 @@
|
||||
"""Pure canonical read model for the operator workbench.
|
||||
|
||||
This module deliberately performs no I/O. It projects database source rows and
|
||||
already-computed opportunity decisions into one current work item per process.
|
||||
Persistent tasks, communications, reconciliation records and outbox rows remain
|
||||
untouched and are retained as source evidence.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
import json
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from app.operation_noise import is_low_value_no_opportunity_item, is_noise_operation_item
|
||||
from app.work_center_action_policy import canonical_action_code
|
||||
|
||||
|
||||
WAITING_ACTION_CODES = {
|
||||
"WAIT_PRODUCTION",
|
||||
"WAIT_CUSTOMER",
|
||||
"WAIT_PAYMENT",
|
||||
"WAIT_SUPPLIER",
|
||||
"WAIT_LOGISTICS",
|
||||
"WAIT_SCHEDULED_DATE",
|
||||
}
|
||||
NO_WORK_ACTION_CODES = {"NO_ACTION", "NOT_FOUND"}
|
||||
|
||||
|
||||
def _s(value: Any) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _upper(value: Any) -> str:
|
||||
return _s(value).upper()
|
||||
|
||||
|
||||
def _source_ref(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
metadata = row.get("item_metadata") if isinstance(row.get("item_metadata"), dict) else {}
|
||||
ref = {
|
||||
"source": _s(row.get("source")),
|
||||
"id": _s(row.get("id")),
|
||||
"status": _s(row.get("status")),
|
||||
"action_code": _upper(row.get("action_code")),
|
||||
}
|
||||
source_id_fields = {
|
||||
"task": "task_id",
|
||||
"communication": "communication_id",
|
||||
"reconciliation": "reconciliation_item_id",
|
||||
"outbox": "outbox_id",
|
||||
}
|
||||
if ref["id"] and ref["source"] in source_id_fields:
|
||||
ref[source_id_fields[ref["source"]]] = ref["id"]
|
||||
for key in (
|
||||
"task_id", "communication_id", "message_id", "raw_event_id",
|
||||
"action_run_id", "conversation_id", "reconciliation_item_id",
|
||||
"outbox_id", "source_system", "source_event_id",
|
||||
):
|
||||
value = row.get(key) or metadata.get(key)
|
||||
if value not in (None, ""):
|
||||
ref[key] = str(value)
|
||||
return ref
|
||||
|
||||
|
||||
def _priority_rank(value: Any) -> int:
|
||||
return {"urgente": 0, "critical": 0, "alta": 1, "high": 1, "normal": 2, "baixa": 3}.get(
|
||||
_s(value).lower(), 3
|
||||
)
|
||||
|
||||
|
||||
def _best_priority(rows: Iterable[Mapping[str, Any]], decision: Mapping[str, Any] | None = None) -> str:
|
||||
values = [_s((decision or {}).get("priority"))]
|
||||
values.extend(_s(row.get("priority")) for row in rows)
|
||||
values = [value for value in values if value]
|
||||
return min(values, key=_priority_rank) if values else "normal"
|
||||
|
||||
|
||||
def _canonical_code(value: Any) -> str:
|
||||
code = canonical_action_code(value)
|
||||
return _upper(code)
|
||||
|
||||
|
||||
def _is_waiting(decision: Mapping[str, Any]) -> bool:
|
||||
code = _canonical_code(decision.get("action_code"))
|
||||
# Unknown WAIT_* actions remain visible until explicitly classified. This
|
||||
# favors a safe false positive over silently hiding a new human action.
|
||||
return code in WAITING_ACTION_CODES
|
||||
|
||||
|
||||
def _matching_task(rows: Iterable[Mapping[str, Any]], action_code: str) -> Mapping[str, Any] | None:
|
||||
for row in rows:
|
||||
if row.get("source") == "task" and _canonical_code(row.get("action_code")) == action_code:
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def _outbox_matches_action(row: Mapping[str, Any], action_code: str) -> bool:
|
||||
metadata = row.get("item_metadata") if isinstance(row.get("item_metadata"), dict) else {}
|
||||
if metadata.get("independent_exception") is True or metadata.get("human_intervention_scope") == "independent":
|
||||
return False
|
||||
candidates = {
|
||||
_canonical_code(row.get("action_code")),
|
||||
_canonical_code(metadata.get("action_code")),
|
||||
_canonical_code(metadata.get("task_action_code")),
|
||||
_canonical_code(metadata.get("current_action_code")),
|
||||
}
|
||||
candidates.discard("")
|
||||
return action_code in candidates
|
||||
|
||||
|
||||
def _explicit_task_id(row: Mapping[str, Any]) -> str:
|
||||
metadata = row.get("item_metadata") if isinstance(row.get("item_metadata"), dict) else {}
|
||||
return _s(row.get("task_id") or metadata.get("task_id"))
|
||||
|
||||
|
||||
def _standalone_group_key(
|
||||
row: Mapping[str, Any],
|
||||
*,
|
||||
explicitly_referenced_task_ids: set[str],
|
||||
) -> str:
|
||||
"""Return a conservative grouping key for rows without an opportunity.
|
||||
|
||||
Conversation identity is a safe process boundary, but not enough to prove
|
||||
that different actions are equivalent. Cross-action merging is allowed
|
||||
only through an explicit communication→task reference.
|
||||
"""
|
||||
process_key = _standalone_process_key(row)
|
||||
if not process_key.startswith("conversation:"):
|
||||
return process_key
|
||||
related_task_id = _explicit_task_id(row)
|
||||
if related_task_id:
|
||||
return f"{process_key}:related-task:{related_task_id}"
|
||||
row_id = _s(row.get("id"))
|
||||
if row.get("source") == "task" and row_id in explicitly_referenced_task_ids:
|
||||
return f"{process_key}:related-task:{row_id}"
|
||||
return f"{process_key}:action:{_canonical_code(row.get('action_code')) or 'REVIEW_MANUALLY'}"
|
||||
|
||||
|
||||
def _has_explicit_unresolved_review(rows: Iterable[Mapping[str, Any]]) -> bool:
|
||||
review_codes = {
|
||||
"REVIEW",
|
||||
"REVIEW_MANUALLY",
|
||||
"ASSOCIATE_OPPORTUNITY",
|
||||
"REVIEW_ASSOCIATION",
|
||||
"LINK_DOCUMENT",
|
||||
}
|
||||
review_statuses = {"needs_review", "review_required", "ambiguous", "conflict"}
|
||||
review_flags = {
|
||||
"needs_review",
|
||||
"review_required",
|
||||
"association_review_required",
|
||||
"customer_association_review_required",
|
||||
"opportunity_association_review_required",
|
||||
}
|
||||
for row in rows:
|
||||
metadata = row.get("item_metadata") if isinstance(row.get("item_metadata"), dict) else {}
|
||||
if _canonical_code(row.get("action_code")) in review_codes:
|
||||
return True
|
||||
if _s(row.get("status")).lower() in review_statuses:
|
||||
return True
|
||||
if _s(row.get("opportunity_linking_status")).lower() in review_statuses:
|
||||
return True
|
||||
if any(metadata.get(flag) is True for flag in review_flags):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def opportunity_ids_from_work_seeds(rows: Iterable[Mapping[str, Any]]) -> list[str]:
|
||||
"""Return decision candidates from eligible seeds only."""
|
||||
return sorted({_s(row.get("opportunity_id")) for row in rows if _s(row.get("opportunity_id"))})
|
||||
|
||||
|
||||
def partition_canonical_items(
|
||||
items: Iterable[Mapping[str, Any]], *, display_limit: int
|
||||
) -> dict[str, Any]:
|
||||
all_items = [dict(item) for item in items]
|
||||
actionable = [item for item in all_items if item.get("operational_queue") != "waiting"]
|
||||
waiting = [item for item in all_items if item.get("operational_queue") == "waiting"]
|
||||
return {
|
||||
"actionable_items": actionable,
|
||||
"waiting_items": waiting,
|
||||
"visible_actionable_items": actionable[:display_limit],
|
||||
"visible_waiting_items": waiting[:display_limit],
|
||||
"work_queue_total": len(actionable),
|
||||
"waiting_total": len(waiting),
|
||||
}
|
||||
|
||||
|
||||
def _unique_refs(items: Iterable[Mapping[str, Any]], field: str) -> list[dict[str, Any]]:
|
||||
refs: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for item in items:
|
||||
for ref in item.get(field) or []:
|
||||
value = dict(ref) if isinstance(ref, Mapping) else {"value": str(ref)}
|
||||
source = _s(value.get("source"))
|
||||
source_id = _s(value.get("id"))
|
||||
fingerprint = (
|
||||
f"{source}:{source_id}"
|
||||
if source and source_id
|
||||
else json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
|
||||
)
|
||||
if fingerprint in seen:
|
||||
continue
|
||||
seen.add(fingerprint)
|
||||
refs.append(value)
|
||||
return refs
|
||||
|
||||
|
||||
def _has_task_execution_handle(item: Mapping[str, Any]) -> bool:
|
||||
return item.get("source") == "task" or _s(item.get("href")).startswith("/tasks/")
|
||||
|
||||
|
||||
def _useful_identity(value: Any) -> bool:
|
||||
normalized = _s(value).casefold()
|
||||
return normalized not in {"", "contacto sem identificação", "contacto", "cliente", "geral"}
|
||||
|
||||
|
||||
def _merge_exact_work_items(items: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Enforce one item for an already-resolved process/action identity."""
|
||||
groups: dict[str, list[Mapping[str, Any]]] = defaultdict(list)
|
||||
for item in items:
|
||||
groups[_s(item.get("work_item_key"))].append(item)
|
||||
|
||||
merged_items: list[dict[str, Any]] = []
|
||||
for work_item_key, group in groups.items():
|
||||
# A task-backed item is the best execution handle. Within the same
|
||||
# handle class, retain the highest existing priority.
|
||||
primary = min(
|
||||
group,
|
||||
key=lambda item: (
|
||||
0 if _has_task_execution_handle(item) else 1,
|
||||
_priority_rank(item.get("priority")),
|
||||
),
|
||||
)
|
||||
merged = dict(primary)
|
||||
source_refs = _unique_refs(group, "source_refs")
|
||||
stale_task_refs = _unique_refs(group, "stale_task_refs")
|
||||
merged["source_refs"] = source_refs
|
||||
merged["stale_task_refs"] = stale_task_refs
|
||||
merged["raw_source_count"] = len(source_refs)
|
||||
merged["priority"] = _best_priority(group)
|
||||
merged["created_at"] = max((_s(item.get("created_at")) for item in group), default="")
|
||||
|
||||
for field in ("customer_name", "contact_display_name", "fiscal_customer_name", "customer_email"):
|
||||
if _useful_identity(merged.get(field)):
|
||||
continue
|
||||
replacement = next((item.get(field) for item in group if _useful_identity(item.get(field))), None)
|
||||
if replacement:
|
||||
merged[field] = replacement
|
||||
|
||||
# Defensive fallback for unusual projected fixtures: never discard a
|
||||
# task URL when a task-backed duplicate contains one.
|
||||
if not _s(merged.get("href")).startswith("/tasks/"):
|
||||
task_href = next(
|
||||
(_s(item.get("href")) for item in group if _s(item.get("href")).startswith("/tasks/")),
|
||||
"",
|
||||
)
|
||||
if task_href:
|
||||
merged["href"] = task_href
|
||||
merged_items.append(merged)
|
||||
|
||||
keys = [_s(item.get("work_item_key")) for item in merged_items]
|
||||
if len(keys) != len(set(keys)):
|
||||
raise AssertionError("canonical Operations invariant violated: duplicate work_item_key")
|
||||
return merged_items
|
||||
|
||||
|
||||
def _standalone_process_key(row: Mapping[str, Any]) -> str:
|
||||
source = _s(row.get("source")) or "unknown"
|
||||
source_system = _s(row.get("source_system")) or source
|
||||
conversation_id = _s(row.get("conversation_id"))
|
||||
if conversation_id and source in {"task", "communication"}:
|
||||
return f"conversation:{source_system}:{conversation_id}"
|
||||
if source == "reconciliation":
|
||||
group_key = _s(row.get("process_group_key") or row.get("source_event_id") or row.get("id"))
|
||||
return f"reconciliation:{group_key}"
|
||||
if source == "outbox":
|
||||
return f"integration-exception:{source_system}:{_s(row.get('id'))}"
|
||||
return f"{source}:{_s(row.get('id'))}"
|
||||
|
||||
|
||||
def _standalone_item(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
process_key = _standalone_process_key(row)
|
||||
action_code = _canonical_code(row.get("action_code")) or "REVIEW_MANUALLY"
|
||||
queue = "exception" if row.get("source") == "outbox" else _s(row.get("queue")) or "rever"
|
||||
item = dict(row)
|
||||
item.update({
|
||||
"process_key": process_key,
|
||||
"work_item_key": f"{process_key}:action:{action_code}",
|
||||
"current_action_code": action_code,
|
||||
"action_code": action_code,
|
||||
"operational_queue": queue,
|
||||
"why_human_required": _s(row.get("detail")) or "A intervenção ainda não está associada com segurança a um processo comercial.",
|
||||
"source_refs": [_source_ref(row)],
|
||||
"raw_source_count": 1,
|
||||
})
|
||||
return item
|
||||
|
||||
|
||||
def _merge_standalone_conversation(rows: list[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
# A conversation is a safe identity boundary. Prefer a task as the execution
|
||||
# handle, but retain every communication/message row as evidence.
|
||||
ordered = sorted(rows, key=lambda row: (row.get("source") != "task", _priority_rank(row.get("priority"))))
|
||||
primary = ordered[0]
|
||||
item = _standalone_item(primary)
|
||||
item["source_refs"] = [_source_ref(row) for row in rows]
|
||||
item["raw_source_count"] = len(rows)
|
||||
item["priority"] = _best_priority(rows)
|
||||
return item
|
||||
|
||||
|
||||
def _opportunity_item(
|
||||
opportunity_id: str,
|
||||
rows: list[Mapping[str, Any]],
|
||||
decision: Mapping[str, Any],
|
||||
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
|
||||
action_code = _canonical_code(decision.get("action_code")) or "REVIEW"
|
||||
independent_exceptions: list[dict[str, Any]] = []
|
||||
evidence_rows: list[Mapping[str, Any]] = []
|
||||
for row in rows:
|
||||
if row.get("source") == "outbox" and not row.get("_evidence_only") and not _outbox_matches_action(row, action_code):
|
||||
independent_exceptions.append(_standalone_item(row))
|
||||
else:
|
||||
evidence_rows.append(row)
|
||||
|
||||
if action_code in NO_WORK_ACTION_CODES:
|
||||
if not _has_explicit_unresolved_review(evidence_rows):
|
||||
return None, independent_exceptions
|
||||
# The central decision normally wins, but an explicit unresolved review
|
||||
# marker must not be hidden by NO_ACTION/NOT_FOUND.
|
||||
action_code = "REVIEW"
|
||||
decision = {
|
||||
**dict(decision),
|
||||
"action_code": action_code,
|
||||
"label": "Rever evidência pendente",
|
||||
"description": "A decisão central indica que não há ação, mas existe evidência explicitamente marcada para revisão.",
|
||||
"reason": "Existe revisão humana não resolvida apesar da decisão central sem ação.",
|
||||
"priority": "normal",
|
||||
"target_url": f"/opportunities/{opportunity_id}",
|
||||
}
|
||||
|
||||
matching_task = _matching_task(evidence_rows, action_code)
|
||||
primary = matching_task or next((row for row in evidence_rows if row.get("source") == "task"), None)
|
||||
primary = primary or (evidence_rows[0] if evidence_rows else {})
|
||||
process_key = f"opportunity:{opportunity_id}"
|
||||
waiting = _is_waiting(decision)
|
||||
opportunity_row = next((row for row in evidence_rows if _s(row.get("fiscal_customer_name"))), None)
|
||||
opportunity_row = opportunity_row or next((row for row in evidence_rows if _s(row.get("opportunity_title"))), None)
|
||||
opportunity_row = opportunity_row or primary
|
||||
|
||||
item = dict(primary)
|
||||
item.update({
|
||||
"id": _s((matching_task or primary).get("id")) or opportunity_id,
|
||||
"source": "opportunity",
|
||||
"process_key": process_key,
|
||||
"work_item_key": f"{process_key}:action:{action_code}",
|
||||
"current_action_code": action_code,
|
||||
"action_code": action_code,
|
||||
"title": _s(decision.get("label")) or _s(primary.get("title")) or action_code,
|
||||
"detail": _s(decision.get("description") or decision.get("reason")),
|
||||
"why_human_required": _s(decision.get("reason") or decision.get("description")),
|
||||
"priority": _s(decision.get("priority")) or _best_priority(evidence_rows),
|
||||
"operational_queue": "waiting" if waiting else "do_now",
|
||||
"queue": _s((matching_task or primary).get("queue")) or _s(primary.get("queue")) or "rever",
|
||||
"status": "waiting" if waiting else "pending",
|
||||
"href": _s((matching_task or {}).get("href")) or _s(decision.get("target_url")) or f"/opportunities/{opportunity_id}",
|
||||
"action_label": _s(decision.get("label")) or _s(primary.get("action_label")) or "Abrir",
|
||||
"opportunity_id": opportunity_id,
|
||||
"opportunity_title": _s(opportunity_row.get("opportunity_title")),
|
||||
"customer_name": _s(opportunity_row.get("fiscal_customer_name") or opportunity_row.get("customer_name")),
|
||||
"contact_display_name": _s(opportunity_row.get("fiscal_customer_name") or opportunity_row.get("contact_display_name")),
|
||||
"source_refs": [_source_ref(row) for row in evidence_rows],
|
||||
"raw_source_count": len(evidence_rows),
|
||||
"stale_task_refs": [
|
||||
_source_ref(row) for row in evidence_rows
|
||||
if row.get("source") == "task" and _canonical_code(row.get("action_code")) != action_code
|
||||
],
|
||||
"decision_version": decision.get("decision_version"),
|
||||
"decision": dict(decision),
|
||||
})
|
||||
return item, independent_exceptions
|
||||
|
||||
|
||||
def canonicalize_operations(
|
||||
source_rows: Iterable[Mapping[str, Any]],
|
||||
opportunity_decisions: Mapping[str, Mapping[str, Any]],
|
||||
*,
|
||||
evidence_rows: Iterable[Mapping[str, Any]] | None = None,
|
||||
display_limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Project eligible work seeds into stable, canonical work items.
|
||||
|
||||
``evidence_rows`` can enrich only opportunities already present in the
|
||||
cleaned seed set. Evidence can never select an opportunity or create a
|
||||
standalone/exception item.
|
||||
"""
|
||||
raw_rows = [dict(row) for row in source_rows]
|
||||
clean_rows = [row for row in raw_rows if not is_noise_operation_item(row) and not is_low_value_no_opportunity_item(row)]
|
||||
raw_evidence_rows = [dict(row) for row in (evidence_rows or [])]
|
||||
clean_evidence_rows = [
|
||||
row for row in raw_evidence_rows
|
||||
if not is_noise_operation_item(row) and not is_low_value_no_opportunity_item(row)
|
||||
]
|
||||
by_opportunity: dict[str, list[Mapping[str, Any]]] = defaultdict(list)
|
||||
standalone_groups: dict[str, list[Mapping[str, Any]]] = defaultdict(list)
|
||||
explicitly_referenced_task_ids = {
|
||||
_explicit_task_id(row) for row in clean_rows
|
||||
if row.get("source") == "communication" and _explicit_task_id(row)
|
||||
}
|
||||
for row in clean_rows:
|
||||
opportunity_id = _s(row.get("opportunity_id"))
|
||||
if opportunity_id:
|
||||
by_opportunity[opportunity_id].append(row)
|
||||
else:
|
||||
standalone_groups[_standalone_group_key(
|
||||
row, explicitly_referenced_task_ids=explicitly_referenced_task_ids
|
||||
)].append(row)
|
||||
|
||||
# Crucial eligibility boundary: evidence-only opportunity IDs are ignored.
|
||||
# Mark retained rows so downstream exception handling cannot turn evidence
|
||||
# into a second work item.
|
||||
selected_opportunity_ids = set(by_opportunity)
|
||||
retained_evidence_rows = []
|
||||
for row in clean_evidence_rows:
|
||||
opportunity_id = _s(row.get("opportunity_id"))
|
||||
if opportunity_id not in selected_opportunity_ids:
|
||||
continue
|
||||
evidence = {**row, "_evidence_only": True}
|
||||
by_opportunity[opportunity_id].append(evidence)
|
||||
retained_evidence_rows.append(evidence)
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
for opportunity_id, rows in by_opportunity.items():
|
||||
decision = opportunity_decisions.get(opportunity_id)
|
||||
if not decision:
|
||||
# A calculation failure must not silently hide customer work.
|
||||
fallback = _merge_standalone_conversation(rows)
|
||||
fallback["process_key"] = f"opportunity:{opportunity_id}"
|
||||
fallback["work_item_key"] = f"opportunity:{opportunity_id}:action:REVIEW"
|
||||
fallback["current_action_code"] = fallback["action_code"] = "REVIEW"
|
||||
fallback["why_human_required"] = "Não foi possível calcular com segurança a ação atual da oportunidade."
|
||||
fallback["opportunity_id"] = opportunity_id
|
||||
items.append(fallback)
|
||||
continue
|
||||
item, exceptions = _opportunity_item(opportunity_id, rows, decision)
|
||||
if item is not None:
|
||||
items.append(item)
|
||||
items.extend(exceptions)
|
||||
|
||||
for rows in standalone_groups.values():
|
||||
# Only a reliable conversation identity is merged. Other process keys are
|
||||
# record-specific and therefore remain separate conservatively.
|
||||
items.append(_merge_standalone_conversation(rows))
|
||||
|
||||
# Relationship-aware standalone grouping may intentionally produce separate
|
||||
# intermediate groups. Exact process/action identity is nevertheless the
|
||||
# final safe uniqueness boundary.
|
||||
items = _merge_exact_work_items(items)
|
||||
|
||||
# Preserve the existing Operations ordering: newest first inside each
|
||||
# priority band, without allowing supporting/stale rows to set the band.
|
||||
items.sort(key=lambda item: _s(item.get("created_at")), reverse=True)
|
||||
items.sort(key=lambda item: _priority_rank(item.get("priority")))
|
||||
all_items = items
|
||||
visible_items = all_items[: max(0, int(display_limit))] if display_limit is not None else all_items
|
||||
return {
|
||||
"items": visible_items,
|
||||
"all_items": all_items,
|
||||
"raw_source_count": len(raw_rows) + len(raw_evidence_rows),
|
||||
"clean_source_count": len(clean_rows) + len(retained_evidence_rows),
|
||||
"seed_source_count": len(clean_rows),
|
||||
"evidence_source_count": len(retained_evidence_rows),
|
||||
"canonical_count": len(all_items),
|
||||
"visible_count": len(visible_items),
|
||||
}
|
||||
@@ -21,6 +21,13 @@ from app.work_center_action_policy import (
|
||||
canonical_action_code,
|
||||
reconstructed_review_required,
|
||||
)
|
||||
from app.canonical_operations import (
|
||||
canonicalize_operations,
|
||||
opportunity_ids_from_work_seeds,
|
||||
partition_canonical_items,
|
||||
)
|
||||
from app.opportunity_next_action_service import get_opportunity_next_actions
|
||||
from app.document_reconciliation_service import active_document_link_exclusion_sql
|
||||
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
@@ -303,6 +310,11 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
|
||||
Technical lists remain available in their own pages and should only appear
|
||||
here when they block an operator action.
|
||||
"""
|
||||
display_limit = max(1, min(int(limit), 200))
|
||||
# Bound each entity source independently, then apply the user-facing limit
|
||||
# after canonicalization. This prevents duplicate rows from consuming the
|
||||
# display limit while still avoiding an unbounded history scan.
|
||||
candidate_limit = max(200, min(display_limit * 20, 1000))
|
||||
with engine.begin() as conn:
|
||||
counts = conn.execute(text("""
|
||||
SELECT
|
||||
@@ -383,7 +395,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
|
||||
LIMIT :limit
|
||||
"""), {"limit": int(limit)}).mappings().all()
|
||||
|
||||
work_items = conn.execute(text("""
|
||||
work_seed_rows = conn.execute(text("""
|
||||
SELECT * FROM (
|
||||
SELECT 'task' AS source, t.id::text AS id, t.created_at, t.due_at,
|
||||
CASE
|
||||
@@ -433,14 +445,19 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
|
||||
COALESCE(o.stage, '') AS opportunity_stage,
|
||||
COALESCE(o.value_amount, 0) AS opportunity_value_amount,
|
||||
COALESCE(o.currency, 'EUR') AS opportunity_currency
|
||||
FROM tasks t
|
||||
FROM (
|
||||
SELECT * FROM tasks
|
||||
WHERE status = 'pending'
|
||||
AND NOT (action_code LIKE 'FOLLOW_UP_%' AND due_at IS NOT NULL AND due_at > now())
|
||||
ORDER BY created_at DESC
|
||||
LIMIT :candidate_limit
|
||||
) t
|
||||
LEFT JOIN opportunities o ON o.id = t.opportunity_id
|
||||
LEFT JOIN customers cu_opp ON cu_opp.id = o.local_customer_id
|
||||
LEFT JOIN customers cu_task ON cu_task.id::text = t.customer_id
|
||||
LEFT JOIN messages m ON m.id = t.message_id
|
||||
LEFT JOIN raw_events re ON re.id = t.raw_event_id
|
||||
WHERE t.status = 'pending'
|
||||
AND NOT (t.action_code LIKE 'FOLLOW_UP_%' AND t.due_at IS NOT NULL AND t.due_at > now())
|
||||
WHERE TRUE
|
||||
|
||||
UNION ALL
|
||||
|
||||
@@ -477,11 +494,16 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
|
||||
COALESCE(o.stage, '') AS opportunity_stage,
|
||||
COALESCE(o.value_amount, 0) AS opportunity_value_amount,
|
||||
COALESCE(o.currency, 'EUR') AS opportunity_currency
|
||||
FROM integration_outbox io
|
||||
FROM (
|
||||
SELECT * FROM integration_outbox
|
||||
WHERE status IN ('failed','blocked')
|
||||
AND NOT (status = 'failed' AND (COALESCE(last_error,'') ILIKE '%limpo manualmente%' OR COALESCE(last_error,'') ILIKE '%resolvido manualmente%'))
|
||||
ORDER BY created_at DESC
|
||||
LIMIT :candidate_limit
|
||||
) io
|
||||
LEFT JOIN opportunities o ON o.id::text = NULLIF(io.payload->>'opportunity_id','')
|
||||
LEFT JOIN customers cu ON cu.id = o.local_customer_id
|
||||
WHERE io.status IN ('pending','failed','blocked')
|
||||
AND NOT (io.status = 'failed' AND (COALESCE(io.last_error,'') ILIKE '%limpo manualmente%' OR COALESCE(io.last_error,'') ILIKE '%resolvido manualmente%'))
|
||||
WHERE TRUE
|
||||
|
||||
UNION ALL
|
||||
|
||||
@@ -519,29 +541,139 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
|
||||
'/communications/' || c.id::text AS href,
|
||||
CASE WHEN c.customer_id IS NULL THEN 'Associar cliente' ELSE 'Abrir' END AS action_label,
|
||||
''::text AS opportunity_linking_status,
|
||||
COALESCE(c.metadata, '{}'::jsonb) AS item_metadata,
|
||||
COALESCE(c.metadata, '{}'::jsonb)
|
||||
|| CASE WHEN c.task_id IS NOT NULL
|
||||
THEN jsonb_build_object('task_id', c.task_id::text)
|
||||
ELSE '{}'::jsonb END AS item_metadata,
|
||||
COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
|
||||
COALESCE(o.stage, '') AS opportunity_stage,
|
||||
COALESCE(o.value_amount, 0) AS opportunity_value_amount,
|
||||
COALESCE(o.currency, 'EUR') AS opportunity_currency
|
||||
FROM communications c
|
||||
FROM (
|
||||
SELECT * FROM communications
|
||||
WHERE status IN ('new','classified','needs_review')
|
||||
ORDER BY created_at DESC
|
||||
LIMIT :candidate_limit
|
||||
) c
|
||||
LEFT JOIN customers cu ON cu.id = c.customer_id
|
||||
LEFT JOIN opportunities o ON o.id = c.opportunity_id
|
||||
WHERE c.status IN ('new','classified','needs_review')
|
||||
WHERE TRUE
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT 'reconciliation' AS source, ri.id::text AS id, ri.created_at, NULL::timestamptz AS due_at,
|
||||
COALESCE(ri.priority, 'normal') AS priority,
|
||||
'rever' AS queue,
|
||||
upper(COALESCE(ri.suggested_action, 'RECONCILE_DOCUMENTS')) AS action_code,
|
||||
COALESCE(ri.title, 'Rever reconciliação') AS title,
|
||||
COALESCE(ri.description, ri.resolution_note, '') AS detail,
|
||||
ri.status,
|
||||
ri.source_system,
|
||||
NULL::text AS conversation_id,
|
||||
NULL::text AS contact_id,
|
||||
COALESCE(ri.description, '') AS request_text,
|
||||
ri.opportunity_id::text,
|
||||
COALESCE(o.title, '') AS opportunity_title,
|
||||
COALESCE(cu.name, ri.customer_name, ri.customer_email, '') AS customer_name,
|
||||
COALESCE(ri.customer_name, '') AS sender_name,
|
||||
COALESCE(ri.customer_email, '') AS sender_email,
|
||||
COALESCE(cu.name, '') AS fiscal_customer_name,
|
||||
COALESCE(cu.email, '') AS fiscal_customer_email,
|
||||
COALESCE(cu.tax_id, '') AS fiscal_customer_tax_id,
|
||||
COALESCE(cu.street_name, '') AS fiscal_customer_street_name,
|
||||
COALESCE(cu.postal_zone, '') AS fiscal_customer_postal_zone,
|
||||
COALESCE(cu.city_name, '') AS fiscal_customer_city_name,
|
||||
COALESCE(cu.name, ri.customer_name, ri.customer_email, '') AS contact_display_name,
|
||||
COALESCE(ri.customer_email, '') AS customer_email,
|
||||
''::text AS no_opportunity_reason,
|
||||
'/reconciliation' AS href,
|
||||
'Rever' AS action_label,
|
||||
''::text AS opportunity_linking_status,
|
||||
COALESCE(ri.payload, '{}'::jsonb) AS item_metadata,
|
||||
COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
|
||||
COALESCE(o.stage, '') AS opportunity_stage,
|
||||
COALESCE(o.value_amount, ri.amount, 0) AS opportunity_value_amount,
|
||||
COALESCE(o.currency, ri.currency, 'EUR') AS opportunity_currency
|
||||
FROM (
|
||||
SELECT * FROM reconciliation_items
|
||||
WHERE status IN ('open','needs_review','conflict')
|
||||
AND """ + active_document_link_exclusion_sql("reconciliation_items") + """
|
||||
ORDER BY created_at DESC
|
||||
LIMIT :candidate_limit
|
||||
) ri
|
||||
LEFT JOIN opportunities o ON o.id = ri.opportunity_id
|
||||
LEFT JOIN customers cu ON cu.id = COALESCE(o.local_customer_id, ri.customer_id)
|
||||
WHERE TRUE
|
||||
) items
|
||||
ORDER BY
|
||||
CASE lower(priority) WHEN 'alta' THEN 1 WHEN 'high' THEN 1 WHEN 'urgente' THEN 0 WHEN 'normal' THEN 2 ELSE 3 END,
|
||||
created_at DESC
|
||||
LIMIT :limit
|
||||
"""), {"limit": int(limit)}).mappings().all()
|
||||
"""), {"candidate_limit": candidate_limit}).mappings().all()
|
||||
|
||||
cleaned_work_items = _attach_operation_urls(_normalise_work_item_intent([dict(r) for r in work_items]))
|
||||
seed_opportunity_ids = opportunity_ids_from_work_seeds(work_seed_rows)
|
||||
evidence_rows = []
|
||||
if seed_opportunity_ids:
|
||||
evidence_rows = conn.execute(text("""
|
||||
SELECT * FROM (
|
||||
SELECT 'communication' AS source, c.id::text AS id, c.created_at,
|
||||
c.status, upper(COALESCE(c.classification, 'REVIEW_MANUALLY')) AS action_code,
|
||||
c.opportunity_id::text, c.source_system, c.conversation_id,
|
||||
COALESCE(c.metadata, '{}'::jsonb)
|
||||
|| CASE WHEN c.task_id IS NOT NULL
|
||||
THEN jsonb_build_object('task_id', c.task_id::text)
|
||||
ELSE '{}'::jsonb END AS item_metadata
|
||||
FROM communications c
|
||||
WHERE c.opportunity_id = ANY(CAST(:opportunity_ids AS UUID[]))
|
||||
AND c.status NOT IN ('new','classified','needs_review')
|
||||
ORDER BY c.created_at DESC
|
||||
LIMIT :candidate_limit
|
||||
) communication_evidence
|
||||
UNION ALL
|
||||
SELECT * FROM (
|
||||
SELECT 'reconciliation' AS source, ri.id::text AS id, ri.created_at,
|
||||
ri.status, upper(COALESCE(ri.suggested_action, 'RECONCILE_DOCUMENTS')) AS action_code,
|
||||
ri.opportunity_id::text, ri.source_system, NULL::text AS conversation_id,
|
||||
COALESCE(ri.payload, '{}'::jsonb) AS item_metadata
|
||||
FROM reconciliation_items ri
|
||||
WHERE ri.opportunity_id = ANY(CAST(:opportunity_ids AS UUID[]))
|
||||
AND ri.status IN ('linked','resolved')
|
||||
ORDER BY ri.created_at DESC
|
||||
LIMIT :candidate_limit
|
||||
) reconciliation_evidence
|
||||
"""), {
|
||||
"opportunity_ids": seed_opportunity_ids,
|
||||
"candidate_limit": candidate_limit,
|
||||
}).mappings().all()
|
||||
|
||||
normalized_rows = _normalise_work_item_intent([dict(r) for r in work_seed_rows])
|
||||
opportunity_ids = opportunity_ids_from_work_seeds(normalized_rows)
|
||||
decisions = get_opportunity_next_actions(opportunity_ids) if opportunity_ids else {}
|
||||
projection = canonicalize_operations(normalized_rows, decisions, evidence_rows=[dict(row) for row in evidence_rows])
|
||||
canonical_items = _attach_operation_urls(list(projection["items"]))
|
||||
partition = partition_canonical_items(canonical_items, display_limit=display_limit)
|
||||
actionable_items = partition["actionable_items"]
|
||||
all_waiting_items = partition["waiting_items"]
|
||||
cleaned_work_items = partition["visible_actionable_items"]
|
||||
waiting_items = partition["visible_waiting_items"]
|
||||
cleaned_counts = {k: _int(v) for k, v in dict(counts).items()}
|
||||
# v4.9.0: the visible Operations total should match the queue the
|
||||
# operator can actually act on, not raw pending tasks that include mailbox
|
||||
# noise awaiting cleanup. The cleanup script still fixes the data source.
|
||||
cleaned_counts["work_queue_total"] = len(cleaned_work_items)
|
||||
cleaned_counts["work_queue_total"] = partition["work_queue_total"]
|
||||
cleaned_counts["waiting_total"] = partition["waiting_total"]
|
||||
cleaned_counts["raw_source_count"] = projection["raw_source_count"]
|
||||
cleaned_counts["canonical_total"] = len(canonical_items)
|
||||
|
||||
diagnostics = {
|
||||
"candidate_limit": candidate_limit,
|
||||
"raw_source_count": projection["raw_source_count"],
|
||||
"clean_source_count": projection["clean_source_count"],
|
||||
"seed_source_count": projection["seed_source_count"],
|
||||
"evidence_source_count": projection["evidence_source_count"],
|
||||
"canonical_count": len(canonical_items),
|
||||
"visible_count": len(cleaned_work_items),
|
||||
"waiting_total": len(all_waiting_items),
|
||||
}
|
||||
return {
|
||||
"counts": cleaned_counts,
|
||||
"recent_outbox": [dict(r) for r in recent_outbox],
|
||||
@@ -550,6 +682,9 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
|
||||
"incomplete_customers": [dict(r) for r in incomplete_customers],
|
||||
"recent_communications": [dict(r) for r in recent_communications],
|
||||
"work_items": cleaned_work_items,
|
||||
"waiting_items": waiting_items,
|
||||
**diagnostics,
|
||||
"projection_metrics": diagnostics,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -302,8 +302,9 @@ def get_opportunity_next_actions(opportunity_ids: Iterable[str]) -> Dict[str, Di
|
||||
if not ids:
|
||||
return {}
|
||||
|
||||
from app.operation_service import ensure_operation_schema
|
||||
ensure_operation_schema()
|
||||
# Read path only: schema creation belongs to startup/migrations. In
|
||||
# particular, GET /operations must never perform DDL while calculating its
|
||||
# canonical projection.
|
||||
with engine.begin() as conn:
|
||||
opportunities = _bulk_rows(conn, """
|
||||
SELECT id::text, stage, status, title,
|
||||
|
||||
Reference in New Issue
Block a user