admin/clientflow_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
261d3420571bbfc1ccbc32dcd6cc3f9449bee73f
clientflow_backend/app/reconciliation_service.py

3377 lines
158 KiB
Python
Raw Blame History

"""Operational reconciliation and external intake service.
v4.9.1 introduces a controlled staging area for information that exists
outside ClientFlow: Jasmin documents, Odoo sales, manual WhatsApp/phone/email
requests and payment proofs. The service deliberately creates reconciliation
candidates first; critical process changes still need operator confirmation.
"""
from __future__ import annotations
import json
import uuid
import logging
import re
import unicodedata
from datetime import datetime
from decimal import Decimal, InvalidOperation
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy import text
from app.action_catalog import get_action_config
from app.db import engine
from app.work_center_action_policy import (
RECONSTRUCTED_SENSITIVE_ACTIONS,
canonical_action_code,
reconstructed_review_metadata_patch,
reconstructed_review_status,
)
_SCHEMA_READY = False
logger = logging.getLogger(__name__)
RECONCILIATION_OPEN_STATUSES = {"open", "needs_review", "conflict"}
DOCUMENT_EXTERNAL_TYPES = {"jasmin_quotation", "jasmin_invoice", "jasmin_proforma", "odoo_sale_order", "packlink_shipment"}
PAYMENT_EXTERNAL_TYPES = {"payment_proof", "payment_receipt"}
STAGE_BY_EXTERNAL_TYPE = {
"jasmin_quotation": "QUOTE_SENT",
"jasmin_proforma": "WAITING_PAYMENT",
"jasmin_invoice": "INVOICE_SENT",
"odoo_sale_order": "ODOO_ORDER_CREATED",
"packlink_shipment": "SHIPMENT_CREATED",
"payment_proof": "WAITING_PAYMENT",
"manual_request": "QUOTE_REQUESTED",
}
NEXT_ACTION_BY_EXTERNAL_TYPE = {
"jasmin_quotation": "SEND_PROFORMA",
"jasmin_proforma": "CONFIRM_PAYMENT",
"jasmin_invoice": "CONFIRM_PAYMENT",
"odoo_sale_order": "SEND_INVOICE",
"packlink_shipment": "REVIEW_MANUALLY",
"payment_proof": "CONFIRM_PAYMENT",
"manual_request": "SEND_QUOTE",
}
def _json(value: Any) -> str:
return json.dumps(value or {}, ensure_ascii=False, default=str)
def _clean(value: Any) -> str:
return str(value or "").strip()
def _uuid_or_none(value: Any) -> Optional[str]:
value = _clean(value)
return value or None
def _money_or_none(value: Any) -> Optional[str]:
raw = _clean(value).replace("€", "").replace(" ", "").replace(",", ".")
if not raw:
return None
try:
return str(Decimal(raw).quantize(Decimal("0.01")))
except (InvalidOperation, ValueError):
return None
def _normalize_tax_id(value: Any) -> str:
"""Normalize NIF/VAT values for exact matching across external systems."""
raw = _clean(value).upper().replace(" ", "").replace("-", "").replace(".", "")
if raw.startswith("PT"):
raw = raw[2:]
return raw
def _strip_accents(value: Any) -> str:
text_value = unicodedata.normalize("NFKD", _clean(value))
return "".join(ch for ch in text_value if not unicodedata.combining(ch))
def _normalize_company_name(value: Any) -> str:
"""Normalize fiscal/company names for identity matching.
Business rule: company fiscal names and NIFs are unique. Therefore an
exact/contained normalized company name is a strong identity signal when a
source omits NIF (common in Odoo). Weak token overlap is still treated only
as a helper for ranking, never as a process bridge.
"""
text_value = _strip_accents(value).upper()
text_value = re.sub(r"[^A-Z0-9]+", " ", text_value)
stop_words = {
"LDA", "LTDA", "LIMITADA", "UNIPESSOAL", "SA", "S", "A",
"SOCIEDADE", "COMERCIAL", "PORTUGAL", "PT", "THE", "COMPANY",
}
tokens = [tok for tok in text_value.split() if tok and tok not in stop_words]
return " ".join(tokens)
def _name_match_score(external_name: Any, *candidate_names: Any) -> int:
external_key = _normalize_company_name(external_name)
if not external_key or len(external_key) < 3:
return 0
external_tokens = set(external_key.split())
best = 0
for candidate in candidate_names:
candidate_key = _normalize_company_name(candidate)
if not candidate_key or len(candidate_key) < 3:
continue
if candidate_key == external_key:
best = max(best, 75)
continue
if external_key in candidate_key or candidate_key in external_key:
best = max(best, 60)
continue
candidate_tokens = set(candidate_key.split())
if external_tokens and candidate_tokens:
overlap = len(external_tokens & candidate_tokens)
ratio = overlap / max(len(external_tokens), len(candidate_tokens))
if overlap >= 2 and ratio >= 0.50:
best = max(best, 30)
elif overlap >= 1 and ratio >= 0.50:
best = max(best, 20)
return best
def _looks_like_company_fiscal_name(value: Any) -> bool:
"""Return true when a name has company/legal-name signals.
Company names are unique in this project, but not every free-text name is a
company fiscal name. This keeps personal contacts such as "Bruno Oliveira"
out of the high-confidence path while allowing "ACZCO BRAGA ENERGY, LDA"
or Odoo display names that include the fiscal name plus location suffix.
"""
text_value = _strip_accents(value).upper()
tokens = set(re.sub(r"[^A-Z0-9]+", " ", text_value).split())
company_tokens = {
"LDA", "LTDA", "LIMITADA", "UNIPESSOAL", "SA", "S", "A",
"SOCIEDADE", "COMERCIAL", "EMPRESA", "ENERGY", "SOLUTIONS",
}
if tokens & company_tokens:
return True
# Typical official/commercial company names often have three or more
# non-address tokens even when the legal suffix was omitted by the source.
normalized = _normalize_company_name(value)
return len(normalized.split()) >= 3
def _process_group_identity_label(match_key: str, items: List[Dict[str, Any]]) -> Tuple[str, str]:
"""Return UI confidence + explanation for a reconstructed process group.
NIF remains absolute. Company fiscal name is also high-confidence by the
project rule that company names are unique; personal/free-text names stay at
medium confidence.
"""
if match_key == "nif":
has_name_bridge = any(
not _normalize_tax_id(item.get("customer_tax_id") or "")
and _looks_like_company_fiscal_name(item.get("customer_name") or item.get("linked_customer_name"))
for item in items
)
if has_name_bridge:
return "alta", "NIF + nome fiscal"
return "alta", "NIF exato"
if match_key == "email":
return "média", "email"
if match_key == "name":
if any(_looks_like_company_fiscal_name(item.get("customer_name") or item.get("linked_customer_name")) for item in items):
return "alta", "nome fiscal"
return "média", "nome"
return "média", match_key or "identidade"
def recent_window_start(days: int = 3) -> str:
"""Inclusive start date for a short operational reconciliation window.
"Últimos 3 dias" means today and the previous two calendar days.
This keeps Reconciliação as a compact work queue instead of a historical import.
"""
from datetime import datetime, timedelta, timezone
days = max(int(days or 1), 1)
return (datetime.now(timezone.utc).date() - timedelta(days=days - 1)).isoformat()
def _priority_for_action(action_code: str, *, fallback: str = "normal") -> str:
code = str(action_code or "").upper()
if code in {"SEND_PROFORMA", "SEND_INVOICE", "CONFIRM_PAYMENT", "PREPARE_ORDER", "VALIDATE_PHYSICAL_ORDER", "CREATE_SHIPMENT"}:
return "alta"
if code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST", "MARK_NO_INTEREST", "IGNORE_SPAM", "NO_ACTION", "IGNORE_BOUNCE"}:
return "baixa"
return fallback
def ensure_reconciliation_schema() -> None:
"""Create additive reconciliation tables.
This is intentionally not a destructive migration: it only creates tables,
indexes and optional columns that allow ClientFlow to stage external data
before an operator links or turns it into a commercial process.
"""
global _SCHEMA_READY
if _SCHEMA_READY:
return
with engine.begin() as conn:
conn.execute(text("""
CREATE TABLE IF NOT EXISTS reconciliation_items (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
source_system TEXT NOT NULL,
external_type TEXT NOT NULL,
external_id TEXT,
title TEXT NOT NULL,
description TEXT,
status TEXT NOT NULL DEFAULT 'open',
priority TEXT NOT NULL DEFAULT 'normal',
suggested_action TEXT,
confidence NUMERIC(4,3),
opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL,
customer_id UUID REFERENCES customers(id) ON DELETE SET NULL,
customer_name TEXT,
customer_email TEXT,
customer_tax_id TEXT,
document_number TEXT,
document_date DATE,
amount NUMERIC(12,2),
currency TEXT NOT NULL DEFAULT 'EUR',
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
resolution_note TEXT,
idempotency_key TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
resolved_at TIMESTAMPTZ
)
"""))
for stmt in [
"ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS suggested_action TEXT",
"ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS confidence NUMERIC(4,3)",
"ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS resolution_note TEXT",
"ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS idempotency_key TEXT",
"ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS customer_tax_id TEXT",
]:
conn.execute(text(stmt))
conn.execute(text("""
CREATE UNIQUE INDEX IF NOT EXISTS ux_reconciliation_items_source_external
ON reconciliation_items(source_system, external_type, external_id)
WHERE external_id IS NOT NULL AND external_id <> ''
"""))
conn.execute(text("""
CREATE UNIQUE INDEX IF NOT EXISTS ux_reconciliation_items_idempotency_key
ON reconciliation_items(idempotency_key)
WHERE idempotency_key IS NOT NULL AND idempotency_key <> ''
"""))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_status ON reconciliation_items(status, created_at DESC)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_type ON reconciliation_items(external_type, status)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_opportunity ON reconciliation_items(opportunity_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_customer_tax_id ON reconciliation_items(customer_tax_id) WHERE customer_tax_id IS NOT NULL AND customer_tax_id <> ''"))
conn.execute(text("""
CREATE TABLE IF NOT EXISTS operation_links (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE,
system TEXT NOT NULL,
external_type TEXT NOT NULL,
external_id TEXT,
external_name TEXT,
external_url TEXT,
status TEXT NOT NULL DEFAULT 'pending',
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
last_synced_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""))
conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS ux_operation_links_key ON operation_links(opportunity_id, system, external_type)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_operation_links_external_id ON operation_links(opportunity_id, system, external_type, external_id)"))
conn.execute(text("""
CREATE TABLE IF NOT EXISTS external_customer_mappings (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
source_system TEXT NOT NULL,
external_party_id TEXT NOT NULL,
external_party_name TEXT,
customer_id UUID REFERENCES customers(id) ON DELETE CASCADE,
confidence TEXT NOT NULL DEFAULT 'confirmed',
created_by TEXT NOT NULL DEFAULT 'operator',
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""))
conn.execute(text("""
CREATE UNIQUE INDEX IF NOT EXISTS ux_external_customer_mappings_source_party
ON external_customer_mappings(source_system, external_party_id)
"""))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_external_customer_mappings_customer ON external_customer_mappings(customer_id)"))
# v4.9.24: store operator decisions so rebuilds do not erase manual
# knowledge about merge/split/link/ignore/history choices.
conn.execute(text("""
CREATE TABLE IF NOT EXISTS reconciliation_decisions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
decision_type TEXT NOT NULL,
status TEXT,
item_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL,
customer_id UUID REFERENCES customers(id) ON DELETE SET NULL,
note TEXT,
actor TEXT NOT NULL DEFAULT 'operator',
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_decisions_type ON reconciliation_decisions(decision_type, created_at DESC)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_decisions_opportunity ON reconciliation_decisions(opportunity_id)"))
# v4.9.13: reconstructed Odoo processes can import the commercial
# lines into the opportunity. Keep the schema guard additive so older
# installs where the product module has not run yet still work.
conn.execute(text("""
CREATE TABLE IF NOT EXISTS opportunity_items (
id UUID PRIMARY KEY,
opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE,
product_id UUID,
sku TEXT,
jasmin_sales_item TEXT,
product_name TEXT NOT NULL,
description TEXT,
quantity NUMERIC(12,2) NOT NULL DEFAULT 1,
unit_price NUMERIC(12,2) NOT NULL DEFAULT 0,
discount_amount NUMERIC(12,2) NOT NULL DEFAULT 0,
total_price NUMERIC(12,2) NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'INTERESTED',
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""))
for stmt in [
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'INTERESTED'",
"CREATE INDEX IF NOT EXISTS idx_opportunity_items_opp ON opportunity_items(opportunity_id)",
]:
conn.execute(text(stmt))
conn.execute(text("""
CREATE TABLE IF NOT EXISTS payment_proofs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL,
customer_id UUID REFERENCES customers(id) ON DELETE SET NULL,
reconciliation_item_id UUID REFERENCES reconciliation_items(id) ON DELETE SET NULL,
source_system TEXT NOT NULL DEFAULT 'manual',
source_ref TEXT,
filename TEXT,
file_url TEXT,
amount NUMERIC(12,2),
currency TEXT NOT NULL DEFAULT 'EUR',
proof_date DATE,
status TEXT NOT NULL DEFAULT 'pending_validation',
note TEXT,
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
validated_at TIMESTAMPTZ,
validated_by TEXT
)
"""))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_payment_proofs_status ON payment_proofs(status, created_at DESC)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_payment_proofs_opportunity ON payment_proofs(opportunity_id)"))
# v4.9.7: keep the reconciliation page resilient on installs where
# the commercial customer/opportunity link migration has not run yet.
# These are additive guards only; they do not change existing data.
conn.execute(text("""
CREATE TABLE IF NOT EXISTS customers (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL DEFAULT '',
tax_id TEXT,
email TEXT,
phone TEXT,
street_name TEXT,
postal_zone TEXT,
city_name TEXT,
country TEXT NOT NULL DEFAULT 'PT',
jasmin_customer_party_key TEXT,
jasmin_customer_id TEXT,
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""))
for stmt in [
"ALTER TABLE customers ADD COLUMN IF NOT EXISTS tax_id TEXT",
"ALTER TABLE customers ADD COLUMN IF NOT EXISTS email TEXT",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS local_customer_id UUID REFERENCES customers(id) ON DELETE SET NULL",
# v4.9.14: the process-link action touches several columns that may
# be missing on long-lived installs where the table was created by
# an older ClientFlow version. Keep all guards additive.
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS stage TEXT NOT NULL DEFAULT 'NEW_LEAD'",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'open'",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS value_amount NUMERIC(12,2)",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS currency TEXT NOT NULL DEFAULT 'EUR'",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS last_action_code TEXT",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb",
"ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS external_id TEXT",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS external_name TEXT",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS external_url TEXT",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS last_synced_at TIMESTAMPTZ",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()",
"ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS task_id UUID",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS action_code TEXT",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS from_stage TEXT",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS to_stage TEXT",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS note TEXT",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS created_by TEXT NOT NULL DEFAULT 'system'",
"ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS product_id UUID",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS sku TEXT",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS product_name TEXT",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS description TEXT",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS quantity NUMERIC(12,2) NOT NULL DEFAULT 1",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS unit_price NUMERIC(12,2) NOT NULL DEFAULT 0",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS discount_amount NUMERIC(12,2) NOT NULL DEFAULT 0",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS total_price NUMERIC(12,2) NOT NULL DEFAULT 0",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'INTERESTED'",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()",
"ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()",
]:
conn.execute(text(stmt))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunities_local_customer ON opportunities(local_customer_id)"))
_SCHEMA_READY = True
def upsert_reconciliation_item(
*,
source_system: str,
external_type: str,
external_id: Optional[str] = None,
title: str,
description: str = "",
status: str = "open",
priority: str = "normal",
suggested_action: Optional[str] = None,
confidence: Optional[float] = None,
opportunity_id: Optional[str] = None,
customer_id: Optional[str] = None,
customer_name: Optional[str] = None,
customer_email: Optional[str] = None,
customer_tax_id: Optional[str] = None,
document_number: Optional[str] = None,
document_date: Optional[str] = None,
amount: Any = None,
currency: str = "EUR",
payload: Optional[Dict[str, Any]] = None,
idempotency_key: Optional[str] = None,
) -> Dict[str, Any]:
ensure_reconciliation_schema()
source_system = _clean(source_system) or "external"
external_type = _clean(external_type) or "external_record"
external_id = _clean(external_id) or None
idempotency_key = _clean(idempotency_key) or (
f"{source_system}:{external_type}:{external_id}" if external_id else None
)
amount_value = _money_or_none(amount)
customer_tax_id = _normalize_tax_id(customer_tax_id) or None
with engine.begin() as conn:
row = conn.execute(text("""
INSERT INTO reconciliation_items (
source_system, external_type, external_id, title, description, status,
priority, suggested_action, confidence, opportunity_id, customer_id,
customer_name, customer_email, customer_tax_id, document_number, document_date, amount,
currency, payload, idempotency_key, updated_at
) VALUES (
:source_system, :external_type, :external_id, :title, :description, :status,
:priority, :suggested_action, :confidence, CAST(:opportunity_id AS UUID),
CAST(:customer_id AS UUID), :customer_name, :customer_email, :customer_tax_id, :document_number,
CAST(:document_date AS DATE), :amount, :currency, CAST(:payload AS JSONB),
:idempotency_key, now()
)
ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL AND idempotency_key <> ''
DO UPDATE SET
title = EXCLUDED.title,
description = EXCLUDED.description,
priority = EXCLUDED.priority,
suggested_action = EXCLUDED.suggested_action,
confidence = EXCLUDED.confidence,
opportunity_id = COALESCE(reconciliation_items.opportunity_id, EXCLUDED.opportunity_id),
customer_id = COALESCE(reconciliation_items.customer_id, EXCLUDED.customer_id),
customer_name = COALESCE(EXCLUDED.customer_name, reconciliation_items.customer_name),
customer_email = COALESCE(EXCLUDED.customer_email, reconciliation_items.customer_email),
customer_tax_id = COALESCE(EXCLUDED.customer_tax_id, reconciliation_items.customer_tax_id),
document_number = COALESCE(EXCLUDED.document_number, reconciliation_items.document_number),
document_date = COALESCE(EXCLUDED.document_date, reconciliation_items.document_date),
amount = COALESCE(EXCLUDED.amount, reconciliation_items.amount),
currency = COALESCE(EXCLUDED.currency, reconciliation_items.currency),
status = CASE
WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN EXCLUDED.status
ELSE reconciliation_items.status
END,
resolution_note = CASE
WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN NULL
ELSE reconciliation_items.resolution_note
END,
resolved_at = CASE
WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN NULL
ELSE reconciliation_items.resolved_at
END,
payload = (CASE
WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN reconciliation_items.payload - 'window_cleanup'
ELSE reconciliation_items.payload
END) || EXCLUDED.payload,
updated_at = now()
RETURNING id::text, source_system, external_type, external_id, title, description,
status, priority, suggested_action, confidence, opportunity_id::text,
customer_id::text, customer_name, customer_email, customer_tax_id, document_number,
document_date, amount, currency, payload, resolution_note, created_at,
updated_at, resolved_at
"""), {
"source_system": source_system,
"external_type": external_type,
"external_id": external_id,
"title": _clean(title) or "Item de reconciliação",
"description": _clean(description),
"status": _clean(status) or "open",
"priority": _clean(priority) or "normal",
"suggested_action": _clean(suggested_action) or None,
"confidence": confidence,
"opportunity_id": _uuid_or_none(opportunity_id),
"customer_id": _uuid_or_none(customer_id),
"customer_name": _clean(customer_name) or None,
"customer_email": _clean(customer_email) or None,
"customer_tax_id": customer_tax_id,
"document_number": _clean(document_number) or None,
"document_date": _clean(document_date) or None,
"amount": amount_value,
"currency": _clean(currency) or "EUR",
"payload": _json(payload or {}),
"idempotency_key": idempotency_key,
}).mappings().first()
return dict(row or {})
def list_reconciliation_items(*, status: str = "open", external_type: Optional[str] = None, limit: int = 200, days: Optional[int] = None) -> List[Dict[str, Any]]:
ensure_reconciliation_schema()
where = []
params: Dict[str, Any] = {"limit": int(limit)}
status = _clean(status) or "open"
if status != "all":
where.append("ri.status = :status")
params["status"] = status
if external_type:
where.append("ri.external_type = :external_type")
params["external_type"] = external_type
if days and status in {"open", "needs_review", "all"}:
# Keep the default page focused on the recent operational window.
# Manual/payment-proof items without a document_date stay visible;
# dated external API candidates outside the window move out of view
# until the operator chooses a wider historical filter/export.
where.append("(ri.document_date IS NULL OR ri.document_date >= CAST(:recent_cutoff AS DATE))")
params["recent_cutoff"] = recent_window_start(days)
where_sql = "WHERE " + " AND ".join(where) if where else ""
with engine.begin() as conn:
rows = conn.execute(text(f"""
SELECT ri.id::text, ri.source_system, ri.external_type, ri.external_id,
ri.title, ri.description, ri.status, ri.priority, ri.suggested_action,
ri.confidence, ri.opportunity_id::text, ri.customer_id::text,
ri.customer_name, ri.customer_email, ri.customer_tax_id, ri.document_number, ri.document_date,
ri.amount, ri.currency, ri.payload, ri.resolution_note, ri.created_at,
ri.updated_at, ri.resolved_at,
o.title AS opportunity_title,
c.name AS linked_customer_name
FROM reconciliation_items ri
LEFT JOIN opportunities o ON o.id = ri.opportunity_id
LEFT JOIN customers c ON c.id = ri.customer_id
{where_sql}
ORDER BY
CASE ri.priority WHEN 'alta' THEN 0 WHEN 'high' THEN 0 WHEN 'normal' THEN 1 ELSE 2 END,
ri.updated_at DESC,
ri.created_at DESC
LIMIT :limit
"""), params).mappings().all()
items = [dict(row) for row in rows]
for item in items:
if str(item.get("status") or "") in RECONCILIATION_OPEN_STATUSES:
try:
item["operation_suggestions"] = find_open_operation_suggestions_for_reconciliation(item, limit=2)
except Exception as exc: # pragma: no cover - production safety guard
# Suggestion lookup is helpful, but it must never break the
# reconciliation page. If an older database/schema has a gap,
# show the items without suggestions and log the cause.
logger.warning("reconciliation suggestions failed for item %s: %s", item.get("id"), exc)
item["operation_suggestions"] = []
item["operation_suggestions_error"] = str(exc)
else:
item["operation_suggestions"] = []
return items
def get_reconciliation_item(item_id: str) -> Optional[Dict[str, Any]]:
ensure_reconciliation_schema()
with engine.begin() as conn:
row = conn.execute(text("""
SELECT id::text, source_system, external_type, external_id, title, description,
status, priority, suggested_action, confidence, opportunity_id::text,
customer_id::text, customer_name, customer_email, customer_tax_id, document_number,
document_date, amount, currency, payload, resolution_note, created_at,
updated_at, resolved_at
FROM reconciliation_items
WHERE id = CAST(:id AS UUID)
LIMIT 1
"""), {"id": item_id}).mappings().first()
return dict(row) if row else None
def reconciliation_summary(*, days: Optional[int] = None) -> Dict[str, int]:
ensure_reconciliation_schema()
where_sql = ""
params: Dict[str, Any] = {}
if days:
where_sql = "WHERE document_date IS NULL OR document_date >= CAST(:recent_cutoff AS DATE)"
params["recent_cutoff"] = recent_window_start(days)
with engine.begin() as conn:
rows = conn.execute(text(f"""
SELECT status, external_type, COUNT(*) AS total
FROM reconciliation_items
{where_sql}
GROUP BY status, external_type
"""), params).mappings().all()
result: Dict[str, int] = {
"open": 0,
"needs_review": 0,
"conflict": 0,
"historical": 0,
"documents": 0,
"payments": 0,
"external_sales": 0,
"shipments": 0,
"resolved": 0,
"ignored": 0,
}
for row in rows:
status = str(row.get("status") or "")
typ = str(row.get("external_type") or "")
total = int(row.get("total") or 0)
if status in RECONCILIATION_OPEN_STATUSES:
result["open"] += total
if status == "needs_review":
result["needs_review"] += total
if status == "conflict":
result["conflict"] += total
if typ in DOCUMENT_EXTERNAL_TYPES:
result["documents"] += total
if typ in PAYMENT_EXTERNAL_TYPES:
result["payments"] += total
if typ == "odoo_sale_order":
result["external_sales"] += total
if typ == "packlink_shipment":
result["shipments"] += total
if status in {"resolved", "linked"}:
result["resolved"] += total
if status == "ignored":
result["ignored"] += total
if status == "historical":
result["historical"] += total
return result
def cleanup_reconciliation_outside_window(
*,
days: int = 3,
source_system: Optional[str] = None,
limit: int = 1000,
actor: str = "cleanup_reconciliation_window",
) -> Dict[str, Any]:
"""Mark open reconciliation candidates older than the working window as ignored.
The reconciliation page is meant to be a short operational queue, not a
historical import list. This helper keeps recent candidates visible and
moves older API candidates out of the operator workflow without deleting
source records or external documents.
"""
ensure_reconciliation_schema()
from datetime import datetime, timedelta, timezone
days = max(int(days or 3), 1)
limit = min(max(int(limit or 1000), 1), 5000)
source_system = _clean(source_system) or None
cutoff = recent_window_start(days)
params: Dict[str, Any] = {"cutoff": cutoff, "limit": limit}
source_sql = ""
if source_system:
source_sql = "AND source_system = :source_system"
params["source_system"] = source_system
with engine.begin() as conn:
rows = conn.execute(text(f"""
SELECT id::text, source_system, external_type, document_number,
customer_name, document_date, title
FROM reconciliation_items
WHERE status IN ('open', 'needs_review')
{source_sql}
AND document_date IS NOT NULL
AND document_date < CAST(:cutoff AS DATE)
ORDER BY document_date ASC, updated_at DESC
LIMIT :limit
"""), params).mappings().all()
ids = [str(r["id"]) for r in rows]
if ids:
conn.execute(text("""
UPDATE reconciliation_items
SET status = 'ignored',
resolution_note = COALESCE(NULLIF(resolution_note, ''), 'Ignorado automaticamente: fora da janela operacional de reconciliação.'),
resolved_at = now(),
updated_at = now(),
payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB)
WHERE id = ANY(CAST(:ids AS UUID[]))
"""), {
"ids": ids,
"payload": _json({"window_cleanup": {"days": days, "cutoff": cutoff, "actor": actor, "source_system": source_system or "all"}}),
})
return {
"days": days,
"cutoff": cutoff,
"source_system": source_system or "all",
"matched": len(rows),
"ignored": len(ids),
"items": [dict(row) for row in rows[:50]],
}
def reset_generated_reconciliation_items(
*,
days: Optional[int] = None,
sources: Optional[list[str]] = None,
statuses: Optional[list[str]] = None,
limit: int = 1000,
actor: str = "operator_ui",
apply: bool = False,
) -> Dict[str, Any]:
"""Preview or delete generated reconciliation staging rows safely.
This is the UI-safe version of scripts/reset_reconciliation_generated.py.
It only targets generated external candidates and always protects rows that
are already linked to an opportunity. When apply=True, a timestamped backup
table is created before deleting rows.
"""
ensure_reconciliation_schema()
from datetime import datetime, timezone
allowed_sources = {"jasmin", "odoo", "packlink"}
allowed_statuses = {"open", "needs_review", "ignored"}
selected_sources = [str(s) for s in (sources or ["jasmin", "odoo", "packlink"]) if str(s) in allowed_sources]
selected_statuses = [str(s) for s in (statuses or ["open", "needs_review", "ignored"]) if str(s) in allowed_statuses]
if not selected_sources:
selected_sources = ["jasmin", "odoo", "packlink"]
if not selected_statuses:
selected_statuses = ["open", "needs_review", "ignored"]
clauses = [
"source_system = ANY(CAST(:sources AS TEXT[]))",
"status = ANY(CAST(:statuses AS TEXT[]))",
"opportunity_id IS NULL",
]
params: Dict[str, Any] = {"sources": selected_sources, "statuses": selected_statuses}
if days is not None:
cutoff = recent_window_start(max(int(days or 1), 1))
params["cutoff"] = cutoff
clauses.append("(document_date IS NULL OR document_date >= CAST(:cutoff AS DATE))")
where_sql = " AND ".join(clauses)
sample_limit = min(max(int(limit or 1000), 1), 5000)
with engine.begin() as conn:
counts = conn.execute(text(f"""
SELECT source_system, external_type, status, COUNT(*) AS total
FROM reconciliation_items
WHERE {where_sql}
GROUP BY source_system, external_type, status
ORDER BY source_system, external_type, status
"""), params).mappings().all()
rows = conn.execute(text(f"""
SELECT id::text, source_system, external_type, status, document_number,
customer_name, customer_tax_id, document_date, amount, title
FROM reconciliation_items
WHERE {where_sql}
ORDER BY updated_at DESC, created_at DESC
LIMIT :limit
"""), {**params, "limit": sample_limit}).mappings().all()
matched = conn.execute(text(f"SELECT COUNT(*) FROM reconciliation_items WHERE {where_sql}"), params).scalar() or 0
backup_table = None
deleted = 0
if apply and int(matched) > 0:
backup_table = "reconciliation_items_reset_backup_" + datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
conn.execute(text(f"CREATE TABLE {backup_table} AS SELECT * FROM reconciliation_items WHERE {where_sql}"), params)
deleted = conn.execute(text(f"DELETE FROM reconciliation_items WHERE {where_sql}"), params).rowcount or 0
return {
"days": days,
"sources": selected_sources,
"statuses": selected_statuses,
"matched": int(matched),
"deleted": int(deleted),
"backup_table": backup_table,
"counts": [dict(r) for r in counts],
"items": [dict(r) for r in rows],
"actor": actor,
"applied": bool(apply),
}
def _norm_match_value(value: Any) -> str:
"""Normalize loose names/emails for conservative matching suggestions."""
return _clean(value).lower()
def find_open_operation_suggestions_for_reconciliation(item: Dict[str, Any], *, limit: int = 3) -> List[Dict[str, Any]]:
"""Suggest open ClientFlow operations/opportunities that may match an external item.
v4.9.11 makes this lookup resilient and less dependent on NIF. Exact NIF
remains the strongest signal, but if Odoo/Jasmin do not provide VAT/NIF the
system can still suggest by normalized company name or email. It never
links automatically.
"""
ensure_reconciliation_schema()
if not item or item.get("opportunity_id"):
return []
customer_id = _uuid_or_none(item.get("customer_id"))
customer_email = _norm_match_value(item.get("customer_email"))
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
customer_tax_id = _normalize_tax_id(
item.get("customer_tax_id")
or payload.get("customer_tax_id")
or payload.get("tax_id")
or ""
)
customer_name = _clean(item.get("customer_name") or item.get("linked_customer_name"))
odoo_partner_external_id = _odoo_partner_external_id_from_item(item) if str(item.get("source_system") or "") == "odoo" else ""
mapped_customer_id = ""
if odoo_partner_external_id:
try:
with engine.begin() as conn:
mapped_customer_id = _clean(conn.execute(text("""
SELECT customer_id::text
FROM external_customer_mappings
WHERE source_system = 'odoo' AND external_party_id = :external_party_id
LIMIT 1
"""), {"external_party_id": odoo_partner_external_id}).scalar())
except Exception:
mapped_customer_id = ""
amount_value = _money_or_none(item.get("amount"))
amount_decimal: Optional[Decimal] = None
if amount_value is not None:
try:
amount_decimal = Decimal(amount_value)
except (InvalidOperation, ValueError):
amount_decimal = None
limit = min(max(int(limit or 3), 1), 5)
# Keep the old broad guard semantics: if not any([customer_id, customer_tax_id, customer_email, customer_name, amount_value])
if not any([customer_id, customer_tax_id, customer_email, customer_name, amount_decimal is not None]):
return []
# Fetch a bounded set of open opportunities and score in Python. This is
# more robust than a large SQL CASE expression across installations with
# slightly different schemas/data types, and avoids a single bad record
# breaking /reconciliation.
with engine.begin() as conn:
rows = conn.execute(text("""
SELECT
o.id::text AS opportunity_id,
o.title AS opportunity_title,
o.stage,
o.status,
o.customer_name,
o.customer_email,
o.value_amount,
o.local_customer_id::text AS local_customer_id,
o.updated_at,
lc.name AS linked_customer_name,
lc.email AS linked_customer_email,
lc.tax_id AS linked_customer_tax_id,
t.id::text AS task_id,
t.action_code,
t.action,
t.route,
t.created_at AS task_created_at
FROM opportunities o
LEFT JOIN customers lc ON lc.id = o.local_customer_id
LEFT JOIN LATERAL (
SELECT id, action_code, action, route, created_at
FROM tasks
WHERE opportunity_id = o.id AND status = 'pending'
ORDER BY
CASE WHEN action_code IN ('SEND_INVOICE','SEND_PROFORMA','CONFIRM_PAYMENT','SEND_QUOTE') THEN 0 ELSE 1 END,
created_at DESC
LIMIT 1
) t ON true
WHERE o.status = 'open'
ORDER BY
CASE WHEN t.id IS NOT NULL THEN 0 ELSE 1 END,
o.updated_at DESC
LIMIT 250
""")).mappings().all()
suggestions: List[Dict[str, Any]] = []
for row in rows:
score = 0
reasons: List[str] = []
if mapped_customer_id and str(row.get("local_customer_id") or "") == mapped_customer_id:
score += 120
reasons.append("mapeamento Odoo confirmado")
if customer_id and str(row.get("local_customer_id") or "") == customer_id:
score += 60
reasons.append("cliente fiscal")
linked_tax = _normalize_tax_id(row.get("linked_customer_tax_id") or "")
# Equivalent of the old SQL signal: COALESCE(lc.tax_id, '') = :customer_tax_id THEN 90 ELSE 0
# but normalized in Python so PT prefixes/spaces do not break matching.
if customer_tax_id and linked_tax and linked_tax == customer_tax_id:
score += 100
reasons.append("NIF exato")
row_emails = {
_norm_match_value(row.get("customer_email")),
_norm_match_value(row.get("linked_customer_email")),
}
if customer_email and customer_email in row_emails:
score += 45
reasons.append("email")
name_score = _name_match_score(
customer_name,
row.get("linked_customer_name"),
row.get("customer_name"),
row.get("opportunity_title"),
)
name_is_strong = name_score >= 35
if name_score:
score += name_score
reasons.append("nome fiscal" if name_is_strong else "nome parcial")
if amount_decimal is not None and row.get("value_amount") is not None:
try:
row_amount = Decimal(str(row.get("value_amount")))
if row_amount >= (amount_decimal * Decimal("0.90")) and row_amount <= (amount_decimal * Decimal("1.10")):
score += 10
reasons.append("valor aproximado")
except (InvalidOperation, ValueError, TypeError):
pass
if row.get("task_id"):
score += 20
reasons.append("operação aberta")
# NIF/email/customer-id are strong. By project rule, a strong normalized
# company fiscal-name match is also a useful identity signal. Weak token
# overlap must not become a suggestion just because there is a pending
# task or a similar amount.
strong_identity = bool(mapped_customer_id and str(row.get("local_customer_id") or "") == mapped_customer_id) or bool(customer_tax_id and linked_tax == customer_tax_id) or bool(customer_email and customer_email in row_emails) or bool(customer_id and str(row.get("local_customer_id") or "") == customer_id)
if not strong_identity and name_score and not name_is_strong:
continue
threshold = 35 if strong_identity else 40
if score < threshold:
continue
suggestions.append({
"opportunity_id": row.get("opportunity_id"),
"opportunity_title": row.get("opportunity_title") or "Oportunidade",
"customer_name": row.get("linked_customer_name") or row.get("customer_name") or "",
"task_id": row.get("task_id"),
"action_code": row.get("action_code") or "",
"action": row.get("action") or "",
"score": int(score),
"reason": ", ".join(dict.fromkeys(reasons)) or "possível correspondência",
})
suggestions.sort(key=lambda s: int(s.get("score") or 0), reverse=True)
return suggestions[:limit]
# v4.9.9 — process timeline reconstruction
# ------------------------------------------------------------
# Reconciliação is more useful when loose external items are shown as a
# possible commercial process instead of isolated rows. These helpers group
# open candidates by strong identity keys (NIF first, then email/name) and infer
# a conservative timeline/state proposal. They never link or create anything
# until the operator explicitly confirms the action in the UI.
def _payload_record(item: Dict[str, Any]) -> Dict[str, Any]:
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
record = payload.get("record") if isinstance(payload.get("record"), dict) else {}
return record
def _odoo_partner_external_id_from_item(item: Dict[str, Any]) -> str:
record = _payload_record(item)
partner_id = record.get("partner_external_id")
if partner_id:
return _clean(partner_id)
value = record.get("partner_id")
if isinstance(value, (list, tuple)) and value:
return _clean(value[0])
return _clean(value)
def _odoo_fulfilment_from_item(item: Dict[str, Any]) -> Dict[str, Any]:
record = _payload_record(item)
fulfilment = record.get("fulfilment")
if isinstance(fulfilment, dict):
return fulfilment
# Compatibility with older payloads: infer only the invoice pending flag.
invoice_status = _clean(record.get("invoice_status"))
return {
"invoice_status": invoice_status,
"invoice_pending": invoice_status in {"to invoice", "no"},
"physical_status": "order_created",
"label": "Venda criada",
"stage": "ODOO_ORDER_CREATED",
"lines": record.get("order_lines") if isinstance(record.get("order_lines"), list) else [],
"outgoing_pickings": record.get("pickings") if isinstance(record.get("pickings"), list) else [],
}
def _odoo_sale_link_payload(item: Dict[str, Any]) -> Dict[str, Any]:
record = _payload_record(item)
return {
"sale_order": {
"id": record.get("id") or item.get("external_id"),
"name": record.get("name") or item.get("document_number"),
"state": record.get("state"),
"partner_id": record.get("partner_id"),
"partner_external_id": record.get("partner_external_id"),
"partner_name": record.get("partner_name") or item.get("customer_name"),
"amount_total": record.get("amount_total") or item.get("amount"),
"date_order": record.get("date_order") or item.get("document_date"),
"invoice_status": record.get("invoice_status"),
},
"order_lines": record.get("order_lines") if isinstance(record.get("order_lines"), list) else [],
"fulfilment": _odoo_fulfilment_from_item(item),
}
def _upsert_external_customer_mapping_from_process(conn: Any, item: Dict[str, Any], opportunity_id: str, *, actor: str) -> None:
if str(item.get("source_system") or "") != "odoo":
return
external_party_id = _odoo_partner_external_id_from_item(item)
if not external_party_id:
return
row = conn.execute(text("""
SELECT o.local_customer_id::text AS customer_id, c.name AS customer_name
FROM opportunities o
LEFT JOIN customers c ON c.id = o.local_customer_id
WHERE o.id = CAST(:opportunity_id AS UUID)
LIMIT 1
"""), {"opportunity_id": opportunity_id}).mappings().first()
customer_id = _clean(row.get("customer_id") if row else "")
if not customer_id:
return
record = _payload_record(item)
external_party_name = _clean(record.get("partner_name") or item.get("customer_name"))
conn.execute(text("""
INSERT INTO external_customer_mappings (source_system, external_party_id, external_party_name, customer_id, confidence, created_by, payload, updated_at)
VALUES ('odoo', :external_party_id, :external_party_name, CAST(:customer_id AS UUID), 'confirmed', :actor, CAST(:payload AS JSONB), now())
ON CONFLICT (source_system, external_party_id)
DO UPDATE SET customer_id = EXCLUDED.customer_id, external_party_name = EXCLUDED.external_party_name, confidence = 'confirmed', created_by = EXCLUDED.created_by, payload = external_customer_mappings.payload || EXCLUDED.payload, updated_at = now()
"""), {
"external_party_id": external_party_id,
"external_party_name": external_party_name,
"customer_id": customer_id,
"actor": actor,
"payload": _json({"confirmed_from_opportunity_id": opportunity_id, "customer_name": row.get("customer_name") if row else ""}),
})
def _upsert_odoo_operation_links_from_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> None:
if str(item.get("external_type") or "") != "odoo_sale_order":
return
record = _payload_record(item)
fulfilment = _odoo_fulfilment_from_item(item)
sale_name = _clean(record.get("name") or item.get("document_number") or item.get("external_id"))
sale_id = _clean(record.get("id") or item.get("external_id"))
conn.execute(text("""
INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at)
VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'sale_order', :external_id, :external_name, NULL, 'created', CAST(:payload AS JSONB), now(), now())
ON CONFLICT (opportunity_id, system, external_type)
DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now()
"""), {
"opportunity_id": opportunity_id,
"external_id": sale_id,
"external_name": sale_name,
"payload": _json(_odoo_sale_link_payload(item)),
})
conn.execute(text("""
INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at)
VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'physical_status', :external_id, :external_name, NULL, :status, CAST(:payload AS JSONB), now(), now())
ON CONFLICT (opportunity_id, system, external_type)
DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now()
"""), {
"opportunity_id": opportunity_id,
"external_id": sale_id,
"external_name": sale_name,
"status": _clean(fulfilment.get("physical_status")) or "order_created",
"payload": _json({"sale_order": sale_name, **fulfilment}),
})
productions = fulfilment.get("productions") if isinstance(fulfilment.get("productions"), list) else []
if productions:
done_count = sum(1 for production in productions if _clean(production.get("state")) == "done")
status = "done" if done_count == len(productions) else "in_progress"
label = f"{done_count}/{len(productions)} produções concluídas" if done_count != len(productions) else f"{done_count} produções concluídas"
conn.execute(text("""
INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at)
VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'production', :external_id, :external_name, NULL, :status, CAST(:payload AS JSONB), now(), now())
ON CONFLICT (opportunity_id, system, external_type)
DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now()
"""), {
"opportunity_id": opportunity_id,
"external_id": sale_id,
"external_name": label,
"status": status,
"payload": _json({"sale_order": sale_name, "productions": productions}),
})
# v132: Odoo ``assigned`` is only a reserved picking. Never create a
# physical_validation link from it. That evidence is created exclusively
# by completing VALIDATE_PHYSICAL_ORDER (or by an unequivocal Odoo done).
if fulfilment.get("delivery_done"):
conn.execute(text("""
INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at)
VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'physical_validation', :external_id, :external_name, NULL, 'validated', CAST(:payload AS JSONB), now(), now())
ON CONFLICT (opportunity_id, system, external_type)
DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now()
"""), {
"opportunity_id": opportunity_id,
"external_id": sale_id,
"external_name": "Entrega concluída",
"payload": _json({"sale_order": sale_name, **fulfilment}),
})
from app.odoo_delivery_task_reconciliation import reconcile_odoo_delivery_done
reconcile_odoo_delivery_done(
conn,
opportunity_id,
evidence={"sale_order": {"id": sale_id, "name": sale_name}, **fulfilment},
actor="reconciliation_odoo_sync",
upsert_validation=False,
)
elif fulfilment.get("delivery_ready"):
conn.execute(text("""
DELETE FROM operation_links
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND system = 'odoo'
AND external_type = 'physical_validation'
AND status IN ('ready_to_ship','pending')
"""), {"opportunity_id": opportunity_id})
def _odoo_importable_lines(item: Dict[str, Any]) -> List[Dict[str, Any]]:
fulfilment = _odoo_fulfilment_from_item(item)
lines = fulfilment.get("lines") if isinstance(fulfilment.get("lines"), list) else []
result: List[Dict[str, Any]] = []
for line in lines:
if not isinstance(line, dict):
continue
name = _clean(line.get("product_name") or line.get("description"))
if not name:
continue
total = _money_or_none(line.get("price_total")) or "0.00"
unit = _money_or_none(line.get("price_unit")) or "0.00"
qty = _money_or_none(line.get("qty_ordered") or line.get("quantity") or "1") or "1.00"
# Zero-value Odoo delivery carrier lines are operational context, not
# commercial product lines for Jasmin invoicing. Keep them in the
# operation link payload but do not pollute Produtos.
if Decimal(total) == Decimal("0.00") and ("delivery" in name.lower() or "shipping" in name.lower()):
continue
result.append({**line, "product_name": name, "price_total": total, "price_unit": unit, "qty_ordered": qty})
return result
def _odoo_product_sku_from_line(line: Dict[str, Any]) -> Optional[str]:
"""Return the ClientFlow SKU used to map Odoo products to Jasmin items.
The Odoo API usually returns numeric product ids in sale order lines, while
the ClientFlow catalogue stores them as ``ODOO-<id>``. Using the raw id as
SKU (``3``) made imported opportunity lines look unmapped even when the
catalogue had ``ODOO-3 -> CARREGADOR_TRIF_22KW`` configured.
"""
raw = _clean(line.get("product_id") or line.get("odoo_product_id") or line.get("sku"))
if not raw:
return None
if raw.upper().startswith("ODOO-"):
return raw.upper()
if raw.isdigit():
return f"ODOO-{raw}"
return raw
def _normalize_product_name(value: Any) -> str:
text_value = unicodedata.normalize("NFKD", _clean(value)).encode("ascii", "ignore").decode("ascii")
return re.sub(r"[^A-Z0-9]+", "", text_value.upper())
def _resolve_product_mapping_for_odoo_line(conn: Any, line: Dict[str, Any]) -> Dict[str, Any]:
"""Resolve an imported Odoo line against the ClientFlow product catalogue.
Priority:
1. ``products.sku = ODOO-<odoo product_id>``
2. ``products.metadata->odoo->product_id``
3. exact normalized product name fallback
"""
sku = _odoo_product_sku_from_line(line)
product_id_raw = _clean(line.get("product_id"))
row = None
if sku:
row = conn.execute(text("""
SELECT id::text, sku, jasmin_sales_item, name
FROM products
WHERE sku = :sku
LIMIT 1
"""), {"sku": sku}).mappings().first()
if row is None and product_id_raw:
row = conn.execute(text("""
SELECT id::text, sku, jasmin_sales_item, name
FROM products
WHERE metadata->'odoo'->>'product_id' = :product_id
LIMIT 1
"""), {"product_id": product_id_raw}).mappings().first()
if row is None:
wanted = _normalize_product_name(line.get("product_name") or line.get("description"))
if wanted:
for candidate in conn.execute(text("""
SELECT id::text, sku, jasmin_sales_item, name
FROM products
WHERE active IS TRUE
LIMIT 500
""")).mappings().all():
if _normalize_product_name(candidate.get("name")) == wanted:
row = candidate
break
if row is None:
return {"product_id": None, "sku": sku, "jasmin_sales_item": None, "catalog_name": None, "mapping_status": "missing"}
return {
"product_id": row.get("id"),
"sku": row.get("sku") or sku,
"jasmin_sales_item": row.get("jasmin_sales_item"),
"catalog_name": row.get("name"),
"mapping_status": "mapped" if row.get("jasmin_sales_item") else "missing_jasmin",
}
def _upsert_opportunity_items_from_odoo_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> None:
if str(item.get("external_type") or "") != "odoo_sale_order":
return
record = _payload_record(item)
sale_name = _clean(record.get("name") or item.get("document_number") or item.get("external_id"))
for line in _odoo_importable_lines(item):
line_id = _clean(line.get("id") or line.get("product_id") or line.get("product_name"))
qty = _money_or_none(line.get("qty_ordered") or "1") or "1.00"
unit_price = _money_or_none(line.get("price_unit") or "0") or "0.00"
total_price = _money_or_none(line.get("price_total") or "0") or "0.00"
status = "DELIVERED" if float(line.get("qty_delivered") or 0) >= float(line.get("qty_ordered") or 0 or 0) and float(line.get("qty_ordered") or 0) > 0 else "ODOO_IMPORTED"
product_mapping = _resolve_product_mapping_for_odoo_line(conn, line)
conn.execute(text("""
INSERT INTO opportunity_items (
id, opportunity_id, product_id, sku, jasmin_sales_item, product_name, description,
quantity, unit_price, discount_amount, total_price, status, metadata
)
SELECT
CAST(:id AS UUID), CAST(:opportunity_id AS UUID), CAST(:product_id AS UUID), :sku, :jasmin_sales_item, :product_name, :description,
:quantity, :unit_price, 0, :total_price, :status, CAST(:metadata AS JSONB)
WHERE NOT EXISTS (
SELECT 1 FROM opportunity_items
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND metadata->>'source_system' = 'odoo'
AND metadata->>'source_line_id' = :source_line_id
)
"""), {
"id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"product_id": product_mapping.get("product_id"),
"sku": product_mapping.get("sku"),
"jasmin_sales_item": product_mapping.get("jasmin_sales_item"),
"product_name": line.get("product_name"),
"description": line.get("description") or line.get("product_name"),
"quantity": qty,
"unit_price": unit_price,
"total_price": total_price,
"status": status,
"source_line_id": line_id,
"metadata": _json({
"source_system": "odoo",
"source_document": sale_name,
"source_line_id": line_id,
"qty_delivered": line.get("qty_delivered"),
"qty_invoiced": line.get("qty_invoiced"),
"product_id": line.get("product_id"),
"resolved_sku": product_mapping.get("sku"),
"resolved_jasmin_sales_item": product_mapping.get("jasmin_sales_item"),
"product_mapping_status": product_mapping.get("mapping_status"),
"catalog_name": product_mapping.get("catalog_name"),
}),
})
# v4.9.26 — Jasmin document reconstruction
# ------------------------------------------------------------
# When a reconciliation candidate is created from a Jasmin quotation/pro-forma/
# invoice, the opportunity must not become an empty reminder. These helpers
# import the external document itself, copy its commercial lines into
# commercial_document_lines/opportunity_items and update the opportunity value.
def _first_value(record: Dict[str, Any], *keys: str) -> Any:
for key in keys:
if key in record and record.get(key) not in (None, ""):
return record.get(key)
return None
def _date_or_none_value(value: Any) -> Optional[str]:
if value is None:
return None
if isinstance(value, datetime):
return value.date().isoformat()
text_value = _clean(value)
if len(text_value) >= 10 and text_value[4:5] == "-" and text_value[7:8] == "-":
return text_value[:10]
return None
def _jasmin_money_value(value: Any) -> Any:
"""Return a numeric-like value from Jasmin money objects.
Jasmin documents commonly encode money as either a scalar field
(for example ``grossValueAmount``) or as an object like
``{"amount": 199.0, "symbol": "€", ...}``. The previous importer
passed those objects directly to Decimal, which produced null/0 values
even though the payload had prices.
"""
if isinstance(value, dict):
for key in ("amount", "baseAmount", "reportingAmount", "value"):
if value.get(key) not in (None, ""):
return value.get(key)
return None
return value
def _decimal_string(value: Any, *, default: Optional[str] = None, places: str = "0.01") -> Optional[str]:
value = _jasmin_money_value(value)
raw = _clean(value).replace("€", "").replace(" ", "").replace(",", ".")
if raw == "":
return default
try:
return str(Decimal(raw).quantize(Decimal(places)))
except (InvalidOperation, ValueError):
return default
def _jasmin_document_kind_for_item(item: Dict[str, Any]) -> str:
external_type = _clean(item.get("external_type"))
return {
"jasmin_quotation": "quotation",
"jasmin_proforma": "proforma",
"jasmin_invoice": "invoice",
}.get(external_type, "document")
def _jasmin_document_status_for_item(item: Dict[str, Any]) -> str:
record = _payload_record(item)
status = _clean(_first_value(record, "status", "documentStatus", "state", "statusDescription"))
return status or "imported"
def _jasmin_first_money(record: Dict[str, Any], *keys: str) -> Any:
for key in keys:
value = _first_value(record, key)
if value not in (None, ""):
return _jasmin_money_value(value)
return None
def _jasmin_document_totals(item: Dict[str, Any]) -> Dict[str, Optional[str]]:
record = _payload_record(item)
amount = _decimal_string(_jasmin_first_money(
record,
"taxExclusiveAmountAmount",
"grossValueAmount",
"netAmount",
"taxExclusiveAmount",
"goodsAmount",
"grossValue",
"amount",
))
tax_amount = _decimal_string(_jasmin_first_money(
record,
"taxTotalAmount",
"taxAmountAmount",
"taxAmount",
"taxTotal",
"vatAmount",
))
total_amount = _decimal_string(_jasmin_first_money(
record,
"payableAmountAmount",
"lineExtensionAmountAmount",
"totalAmountAmount",
"payableAmount",
"totalAmount",
"total",
"grossAmount",
"lineExtensionAmount",
"amount",
) or item.get("amount"))
if not amount and total_amount:
amount = total_amount
return {"amount": amount, "tax_amount": tax_amount, "total_amount": total_amount}
def _jasmin_line_lists(record: Dict[str, Any]) -> List[Dict[str, Any]]:
result: List[Dict[str, Any]] = []
for key in (
"lines",
"documentLines",
"document_lines",
"documentLine",
"salesLines",
"orderLines",
"quotationLines",
"invoiceLines",
"items",
):
value = record.get(key)
if isinstance(value, list):
result.extend([line for line in value if isinstance(line, dict)])
return result
def _jasmin_sales_item_from_line(line: Dict[str, Any]) -> str:
value = _first_value(
line,
"salesItem",
"salesItemId",
"salesItemKey",
"itemKey",
"itemCode",
"item",
"article",
"productCode",
"sku",
)
if isinstance(value, dict):
value = _first_value(value, "key", "id", "code", "name")
return _clean(value)
def _jasmin_description_from_line(line: Dict[str, Any]) -> str:
value = _first_value(
line,
"description",
"itemDescription",
"salesItemDescription",
"productName",
"name",
"articleDescription",
)
if isinstance(value, dict):
value = _first_value(value, "description", "name", "key")
description = _clean(value)
return description or _jasmin_sales_item_from_line(line) or "Linha Jasmin"
def _jasmin_document_lines_from_item(item: Dict[str, Any]) -> List[Dict[str, Any]]:
record = _payload_record(item)
lines = []
for idx, line in enumerate(_jasmin_line_lists(record)):
description = _jasmin_description_from_line(line)
jasmin_sales_item = _jasmin_sales_item_from_line(line)
quantity = _decimal_string(_first_value(line, "quantity", "qty", "orderedQuantity", "invoicedQuantity"), default="1.000", places="0.001") or "1.000"
unit = _clean(_first_value(line, "unit", "unitKey", "unitOfMeasure", "unitOfMeasureCode")) or "UN"
unit_price = _decimal_string(_jasmin_first_money(
line,
"unitPriceAmount",
"unitPrice",
"priceAmount",
"price",
"unitAmount",
), default="0.00") or "0.00"
total_amount = _decimal_string(_jasmin_first_money(
line,
"lineExtensionAmountAmount",
"totalAmountAmount",
"grossValueAmount",
"taxExclusiveAmountAmount",
"totalAmount",
"lineExtensionAmount",
"grossValue",
"netAmount",
"amount",
"grossAmount",
))
if total_amount is None:
try:
total_amount = str((Decimal(quantity) * Decimal(unit_price)).quantize(Decimal("0.01")))
except (InvalidOperation, ValueError):
total_amount = "0.00"
line_id = _clean(_first_value(line, "id", "lineId", "lineNumber", "naturalKey", "key")) or f"{idx}:{jasmin_sales_item or description}"
tax_schema = _clean(_first_value(line, "taxSchema", "itemTaxSchema", "taxSchemaKey", "vatCode")) or "NORMAL"
lines.append({
"source_line_id": line_id,
"line_index": idx,
"jasmin_sales_item": jasmin_sales_item,
"description": description,
"quantity": quantity,
"unit": unit,
"unit_price": unit_price,
"tax_schema": tax_schema,
"total_amount": total_amount,
"payload": line,
})
return lines
def _resolve_product_mapping_for_jasmin_line(conn: Any, line: Dict[str, Any]) -> Dict[str, Any]:
jasmin_sales_item = _clean(line.get("jasmin_sales_item"))
row = None
if jasmin_sales_item:
row = conn.execute(text("""
SELECT id::text, sku, jasmin_sales_item, name
FROM products
WHERE jasmin_sales_item = :jasmin_sales_item
OR sku = :jasmin_sales_item
LIMIT 1
"""), {"jasmin_sales_item": jasmin_sales_item}).mappings().first()
if row is None:
wanted = _normalize_product_name(line.get("description"))
if wanted:
for candidate in conn.execute(text("""
SELECT id::text, sku, jasmin_sales_item, name
FROM products
WHERE active IS TRUE
LIMIT 500
""")).mappings().all():
if _normalize_product_name(candidate.get("name")) == wanted:
row = candidate
break
if row is None:
return {"product_id": None, "sku": None, "jasmin_sales_item": jasmin_sales_item or None, "catalog_name": None, "mapping_status": "missing"}
return {
"product_id": row.get("id"),
"sku": row.get("sku"),
"jasmin_sales_item": row.get("jasmin_sales_item") or jasmin_sales_item,
"catalog_name": row.get("name"),
"mapping_status": "mapped" if row.get("id") else "missing",
}
def _upsert_jasmin_document_from_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> Optional[str]:
if _clean(item.get("source_system")) != "jasmin" or not _clean(item.get("external_type")).startswith("jasmin_"):
return None
try:
from app.commercial_service import ensure_commercial_schema
ensure_commercial_schema()
except Exception as exc: # pragma: no cover - safety guard
logger.warning("failed to ensure commercial schema for Jasmin import: %s", exc)
return None
record = _payload_record(item)
document_kind = _jasmin_document_kind_for_item(item)
totals = _jasmin_document_totals(item)
external_id = _clean(item.get("external_id") or _first_value(record, "id", "key", "documentKey", "naturalKey"))
document_number = _clean(item.get("document_number") or _first_value(record, "documentNumber", "number", "naturalKey", "name", "reference") or external_id)
customer_id = _uuid_or_none(item.get("customer_id"))
existing = conn.execute(text("""
SELECT id::text
FROM commercial_documents
WHERE system = 'jasmin'
AND (
(CAST(:external_id AS TEXT) <> '' AND external_id = CAST(:external_id AS TEXT))
OR (CAST(:document_number AS TEXT) <> '' AND document_number = CAST(:document_number AS TEXT))
)
AND (opportunity_id = CAST(:opportunity_id AS UUID) OR opportunity_id IS NULL)
ORDER BY opportunity_id NULLS LAST, created_at DESC
LIMIT 1
"""), {"external_id": external_id, "document_number": document_number, "opportunity_id": opportunity_id}).scalar()
payload = {
"source": "reconciliation_jasmin_import",
"reconciliation_item_id": item.get("id"),
"external_type": item.get("external_type"),
"record": record,
}
params = {
"id": existing or str(uuid.uuid4()),
"customer_id": customer_id,
"opportunity_id": opportunity_id,
"document_kind": document_kind,
"external_id": external_id or None,
"company": _clean(_first_value(record, "company", "companyKey")) or None,
"document_type": _clean(_first_value(record, "documentType", "documentTypeKey")) or None,
"serie": _clean(_first_value(record, "serie", "serieKey", "series")) or None,
"series_number": _clean(_first_value(record, "seriesNumber", "sequenceNumber")) or None,
"document_number": document_number or None,
"customer_party_key": _clean(_first_value(record, "buyerCustomerParty", "customerParty", "customerPartyKey", "partyKey")) or None,
"status": _jasmin_document_status_for_item(item),
"amount": totals.get("amount"),
"tax_amount": totals.get("tax_amount"),
"total_amount": totals.get("total_amount"),
"currency": item.get("currency") or _clean(_first_value(record, "currency", "currencyKey", "currencyCode")) or "EUR",
"document_date": _date_or_none_value(item.get("document_date") or _first_value(record, "documentDate", "date", "creationDate", "postingDate")),
"due_date": _date_or_none_value(_first_value(record, "dueDate", "paymentDueDate")),
"payload": _json(payload),
"role": "current" if document_kind in {"quotation", "proforma", "invoice"} else "related",
"is_primary": True,
}
if params["role"] in {"current", "accepted"}:
conn.execute(text("""
UPDATE commercial_documents
SET role = CASE WHEN COALESCE(role, 'current') = 'current' THEN 'historical' ELSE role END,
is_primary = FALSE,
is_active = CASE WHEN COALESCE(role, 'current') = 'current' THEN FALSE ELSE COALESCE(is_active, TRUE) END,
updated_at = now()
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND system = 'jasmin'
AND document_kind = :document_kind
AND id <> CAST(:id AS UUID)
AND COALESCE(role, 'current') IN ('current', 'accepted')
AND COALESCE(is_primary, TRUE) = TRUE
"""), params)
if existing:
conn.execute(text("""
UPDATE commercial_documents
SET customer_id = COALESCE(CAST(:customer_id AS UUID), customer_id),
opportunity_id = CAST(:opportunity_id AS UUID),
document_kind = :document_kind,
external_id = COALESCE(:external_id, external_id),
company = COALESCE(:company, company),
document_type = COALESCE(:document_type, document_type),
serie = COALESCE(:serie, serie),
series_number = CASE WHEN CAST(:series_number AS TEXT) ~ '^[0-9]+$' THEN CAST(:series_number AS INTEGER) ELSE series_number END,
document_number = COALESCE(:document_number, document_number),
customer_party_key = COALESCE(:customer_party_key, customer_party_key),
status = COALESCE(:status, status),
amount = COALESCE(CAST(:amount AS NUMERIC), amount),
tax_amount = COALESCE(CAST(:tax_amount AS NUMERIC), tax_amount),
total_amount = COALESCE(CAST(:total_amount AS NUMERIC), total_amount),
currency = COALESCE(:currency, currency),
document_date = COALESCE(CAST(:document_date AS DATE), document_date),
due_date = COALESCE(CAST(:due_date AS DATE), due_date),
role = :role,
is_primary = :is_primary,
is_active = TRUE,
payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB),
updated_at = now()
WHERE id = CAST(:id AS UUID)
"""), params)
document_id = existing
conn.execute(text("DELETE FROM commercial_document_lines WHERE document_id = CAST(:document_id AS UUID)"), {"document_id": document_id})
else:
conn.execute(text("""
INSERT INTO commercial_documents (
id, customer_id, opportunity_id, system, document_kind, external_id, company,
document_type, serie, series_number, document_number, customer_party_key,
status, amount, tax_amount, total_amount, currency, role, is_primary, document_date, due_date,
payload, updated_at
) VALUES (
CAST(:id AS UUID), CAST(:customer_id AS UUID), CAST(:opportunity_id AS UUID), 'jasmin',
:document_kind, :external_id, :company, :document_type, :serie,
CASE WHEN CAST(:series_number AS TEXT) ~ '^[0-9]+$' THEN CAST(:series_number AS INTEGER) ELSE NULL END,
:document_number, :customer_party_key, :status, CAST(:amount AS NUMERIC),
CAST(:tax_amount AS NUMERIC), CAST(:total_amount AS NUMERIC), :currency,
:role, :is_primary, CAST(:document_date AS DATE), CAST(:due_date AS DATE), CAST(:payload AS JSONB), now()
)
"""), params)
document_id = params["id"]
for line in _jasmin_document_lines_from_item(item):
mapping = _resolve_product_mapping_for_jasmin_line(conn, line)
conn.execute(text("""
INSERT INTO commercial_document_lines (
document_id, line_index, local_product_id, jasmin_sales_item, description,
quantity, unit, unit_price, tax_schema, total_amount, payload
) VALUES (
CAST(:document_id AS UUID), :line_index, CAST(:local_product_id AS UUID), :jasmin_sales_item,
:description, CAST(:quantity AS NUMERIC), :unit, CAST(:unit_price AS NUMERIC),
:tax_schema, CAST(:total_amount AS NUMERIC), CAST(:payload AS JSONB)
)
"""), {
"document_id": document_id,
"line_index": line.get("line_index"),
"local_product_id": mapping.get("product_id"),
"jasmin_sales_item": mapping.get("jasmin_sales_item") or line.get("jasmin_sales_item"),
"description": line.get("description"),
"quantity": line.get("quantity"),
"unit": line.get("unit"),
"unit_price": line.get("unit_price"),
"tax_schema": line.get("tax_schema"),
"total_amount": line.get("total_amount"),
"payload": _json({"source_line_id": line.get("source_line_id"), "mapping": mapping, "raw": line.get("payload")}),
})
return document_id
def _upsert_opportunity_items_from_jasmin_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> int:
if _clean(item.get("source_system")) != "jasmin" or not _clean(item.get("external_type")).startswith("jasmin_"):
return 0
document_ref = _clean(item.get("document_number") or item.get("external_id"))
upserted = 0
for line in _jasmin_document_lines_from_item(item):
mapping = _resolve_product_mapping_for_jasmin_line(conn, line)
source_line_id = _clean(line.get("source_line_id"))
params = {
"id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"product_id": mapping.get("product_id"),
"sku": mapping.get("sku"),
"jasmin_sales_item": mapping.get("jasmin_sales_item") or line.get("jasmin_sales_item"),
"product_name": mapping.get("catalog_name") or line.get("description"),
"description": line.get("description"),
"quantity": line.get("quantity"),
"unit_price": line.get("unit_price"),
"total_price": line.get("total_amount"),
"status": "JASMIN_IMPORTED",
"source_document": document_ref,
"source_line_id": source_line_id,
"metadata": _json({
"source_system": "jasmin",
"source_document": document_ref,
"source_line_id": source_line_id,
"source_external_id": item.get("external_id"),
"source_external_type": item.get("external_type"),
"resolved_sku": mapping.get("sku"),
"resolved_jasmin_sales_item": mapping.get("jasmin_sales_item"),
"product_mapping_status": mapping.get("mapping_status"),
"catalog_name": mapping.get("catalog_name"),
"price_source": "jasmin_document_line",
}),
}
updated = conn.execute(text("""
UPDATE opportunity_items
SET product_id = COALESCE(CAST(:product_id AS UUID), product_id),
sku = COALESCE(:sku, sku),
jasmin_sales_item = COALESCE(:jasmin_sales_item, jasmin_sales_item),
product_name = COALESCE(:product_name, product_name),
description = COALESCE(:description, description),
quantity = CAST(:quantity AS NUMERIC),
unit_price = CAST(:unit_price AS NUMERIC),
total_price = CAST(:total_price AS NUMERIC),
status = CASE WHEN status IN ('REJECTED', 'CANCELLED') THEN status ELSE :status END,
metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB),
updated_at = now()
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND metadata->>'source_system' = 'jasmin'
AND metadata->>'source_document' = :source_document
AND metadata->>'source_line_id' = :source_line_id
"""), params).rowcount or 0
if updated:
upserted += updated
continue
inserted = conn.execute(text("""
INSERT INTO opportunity_items (
id, opportunity_id, product_id, sku, jasmin_sales_item, product_name, description,
quantity, unit_price, discount_amount, total_price, status, metadata
)
SELECT
CAST(:id AS UUID), CAST(:opportunity_id AS UUID), CAST(:product_id AS UUID), :sku,
:jasmin_sales_item, :product_name, :description, CAST(:quantity AS NUMERIC),
CAST(:unit_price AS NUMERIC), 0, CAST(:total_price AS NUMERIC), :status,
CAST(:metadata AS JSONB)
WHERE NOT EXISTS (
SELECT 1 FROM opportunity_items
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND metadata->>'source_system' = 'jasmin'
AND metadata->>'source_document' = :source_document
AND metadata->>'source_line_id' = :source_line_id
)
"""), params).rowcount or 0
upserted += inserted
return upserted
def _apply_jasmin_documents_to_opportunity(conn: Any, items: List[Dict[str, Any]], opportunity_id: str, *, actor: str) -> Dict[str, Any]:
jasmin_items = [item for item in items if _clean(item.get("source_system")) == "jasmin" and _clean(item.get("external_type")).startswith("jasmin_")]
if not jasmin_items:
return {"documents": 0, "lines": 0}
document_count = 0
line_count = 0
product_names: List[str] = []
best_amount = None
for item in jasmin_items:
if _upsert_jasmin_document_from_item(conn, item, opportunity_id):
document_count += 1
line_count += _upsert_opportunity_items_from_jasmin_item(conn, item, opportunity_id)
for line in _jasmin_document_lines_from_item(item):
name = _clean(line.get("description"))
if name and name not in product_names:
product_names.append(name)
totals = _jasmin_document_totals(item)
best_amount = best_amount or totals.get("total_amount") or item.get("amount")
product_interest = ", ".join(product_names[:3])
if len(product_names) > 3:
product_interest += f" +{len(product_names)-3} linha(s)"
conn.execute(text("""
UPDATE opportunities
SET value_amount = CASE
WHEN CAST(:amount AS NUMERIC) IS NOT NULL AND (value_amount IS NULL OR value_amount = 0) THEN CAST(:amount AS NUMERIC)
ELSE value_amount
END,
product_interest = CASE
WHEN :product_interest <> '' AND (product_interest IS NULL OR product_interest = '' OR product_interest ILIKE 'Processo importado%%') THEN :product_interest
ELSE product_interest
END,
metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB),
updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
"""), {
"opportunity_id": opportunity_id,
"amount": _money_or_none(best_amount),
"product_interest": product_interest,
"metadata": _json({
"jasmin_reconciliation_imported": True,
"jasmin_documents_imported": document_count,
"jasmin_lines_imported": line_count,
"jasmin_import_actor": actor,
}),
})
conn.execute(text("""
UPDATE opportunities
SET value_amount = COALESCE((
SELECT SUM(total_price)
FROM opportunity_items
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND status NOT IN ('REJECTED', 'CANCELLED')
), value_amount, 0),
updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
AND (value_amount IS NULL OR value_amount = 0)
AND EXISTS (SELECT 1 FROM opportunity_items WHERE opportunity_id = CAST(:opportunity_id AS UUID))
"""), {"opportunity_id": opportunity_id})
return {"documents": document_count, "lines": line_count, "product_interest": product_interest}
def _ensure_pending_task_for_reconstruction(conn: Any, opportunity_id: str, action_code: str, *, note: str, actor: str) -> None:
action_code = _clean(action_code).upper() or "REVIEW_MANUALLY"
exists = conn.execute(text("""
SELECT id::text FROM tasks
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND status = 'pending'
AND action_code = :action_code
LIMIT 1
"""), {"opportunity_id": opportunity_id, "action_code": action_code}).scalar()
if exists:
return
try:
config = get_action_config(action_code)
conn.execute(text("""
INSERT INTO tasks (
id, opportunity_id, action_code, route, action, note, action_required,
safe_to_post, status, source_system, source_event_id, idempotency_key, metadata, created_at, updated_at
) VALUES (
CAST(:id AS UUID), CAST(:opportunity_id AS UUID), :action_code, :route, :action, :note, TRUE,
FALSE, 'pending', 'reconciliation_process', :source_event_id, :idempotency_key, CAST(:metadata AS JSONB), now(), now()
)
ON CONFLICT (idempotency_key) DO NOTHING
"""), {
"id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"action_code": action_code,
"route": config.get("route") or "financeiro",
"action": config.get("action") or action_code,
"note": note,
"source_event_id": f"apply:{opportunity_id}:{action_code}",
"idempotency_key": f"task:reconciliation_apply:{opportunity_id}:{action_code}",
"metadata": _json({"source": "reconciliation_apply", "actor": actor}),
})
except Exception as exc: # pragma: no cover - production safety guard
logger.warning("failed to create reconstruction task for %s: %s", opportunity_id, exc)
def _apply_reconstructed_process_to_opportunity(conn: Any, items: List[Dict[str, Any]], opportunity_id: str, *, actor: str) -> Dict[str, Any]:
"""Apply reconstructed evidence to the opportunity summary/pipeline.
Linking is no longer just a loose reference: Odoo/Jasmin evidence updates
operation_links, imports Odoo lines into Produtos, writes timeline events
and moves the visual stage to the most advanced reconstructed step.
"""
state = infer_reconciliation_process_state(items)
stage = _clean(state.get("stage")) or "REVIEW"
action_code = _clean(state.get("action_code")) or "REVIEW_MANUALLY"
amount = next((item.get("amount") for item in items if item.get("amount") is not None), None)
amount_value = _money_or_none(amount)
odoo_items = [item for item in items if str(item.get("external_type") or "") == "odoo_sale_order"]
for item in items:
_upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor)
_upsert_odoo_operation_links_from_item(conn, item, opportunity_id)
_upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id)
jasmin_import = _apply_jasmin_documents_to_opportunity(conn, items, opportunity_id, actor=actor)
current = conn.execute(text("""
SELECT stage, last_action_code, COALESCE(metadata, '{}'::jsonb) AS metadata
FROM opportunities
WHERE id = CAST(:opportunity_id AS UUID)
FOR UPDATE
"""), {"opportunity_id": opportunity_id}).mappings().first() or {}
try:
from app.opportunity_service import OPPORTUNITY_STAGE_RANK
current_rank = int(OPPORTUNITY_STAGE_RANK.get(_clean(current.get("stage")), 0))
suggested_rank = int(OPPORTUNITY_STAGE_RANK.get(stage, 0))
except Exception:
current_rank = suggested_rank = 0
preserve_current = bool(_clean(current.get("stage"))) and current_rank >= suggested_rank
effective_stage = _clean(current.get("stage")) if preserve_current else stage
effective_action_code = _clean(current.get("last_action_code")) if preserve_current else action_code
effective_action_code = effective_action_code or action_code
current_review_status = reconstructed_review_status(current.get("metadata"))
requires_review = (
canonical_action_code(action_code) in {canonical_action_code(code) for code in RECONSTRUCTED_SENSITIVE_ACTIONS}
and current_review_status not in {"validated", "waived"}
)
review_patch = (
reconstructed_review_metadata_patch(
"required", actor=actor,
reason="Processo reconstruído com ação sensível pendente.",
blocked_action_code=action_code,
)
if requires_review else {}
)
metadata_payload = {
"reconstruction_applied": True,
"actor": actor,
"suggested_stage": stage,
"suggested_action": action_code,
"effective_stage": effective_stage,
"effective_action": effective_action_code,
"stage_regression_prevented": preserve_current and effective_stage != stage,
"item_ids": [str(item.get("id")) for item in items if item.get("id")],
"jasmin_import": jasmin_import,
"clientflow_record_mode": "reconstructed_invoice_review",
**review_patch,
}
conn.execute(text("""
UPDATE opportunities
SET stage = :stage,
last_action_code = :action_code,
value_amount = CASE
WHEN CAST(:amount AS NUMERIC) IS NOT NULL AND (value_amount IS NULL OR value_amount = 0) THEN CAST(:amount AS NUMERIC)
ELSE value_amount
END,
metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB),
updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
"""), {
"opportunity_id": opportunity_id,
"stage": effective_stage,
"action_code": effective_action_code,
"amount": amount_value,
"metadata": _json(metadata_payload),
})
# If Odoo lines were imported and the opportunity value was still zero,
# recalculate from opportunity_items. This preserves an existing manual
# value when one has already been set.
conn.execute(text("""
UPDATE opportunities
SET value_amount = COALESCE((
SELECT SUM(total_price)
FROM opportunity_items
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND status NOT IN ('REJECTED', 'CANCELLED')
), value_amount, 0),
updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
AND (value_amount IS NULL OR value_amount = 0)
AND EXISTS (SELECT 1 FROM opportunity_items WHERE opportunity_id = CAST(:opportunity_id AS UUID))
"""), {"opportunity_id": opportunity_id})
has_invoice = bool(conn.execute(text("""
SELECT EXISTS (
SELECT 1
FROM commercial_documents
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND document_kind = 'invoice'
AND COALESCE(is_active, TRUE) = TRUE
AND COALESCE(role, 'current') IN ('current', 'accepted', 'historical', 'history')
)
"""), {"opportunity_id": opportunity_id}).scalar())
if odoo_items and effective_action_code == "SEND_INVOICE" and not has_invoice:
_ensure_pending_task_for_reconstruction(
conn,
opportunity_id,
"REVIEW_RECONSTRUCTED_PROCESS" if requires_review else effective_action_code,
note=(
"Validar cliente, documentos e valor do processo reconstruído antes de enviar fatura."
if requires_review
else "Processo reconstruído: validar se falta emitir/enviar fatura antes de avançar."
),
actor=actor,
)
return {
"stage": effective_stage,
"action_code": effective_action_code,
"amount": amount_value,
"stage_regression_prevented": preserve_current and effective_stage != stage,
}
PROCESS_STEP_BY_EXTERNAL_TYPE = {
"manual_request": (5, "Pedido externo registado", "QUOTE_REQUESTED", "SEND_QUOTE"),
"jasmin_quotation": (20, "Orçamento encontrado no Jasmin", "QUOTE_SENT", "SEND_PROFORMA"),
"jasmin_proforma": (30, "Orçamento para pagamento encontrado no Jasmin", "WAITING_PAYMENT", "CONFIRM_PAYMENT"),
"payment_proof": (40, "Comprovativo de pagamento recebido", "WAITING_PAYMENT", "CONFIRM_PAYMENT"),
"jasmin_invoice": (50, "Fatura encontrada no Jasmin", "INVOICE_SENT", "CONFIRM_PAYMENT"),
"odoo_sale_order": (60, "Venda/encomenda encontrada no Odoo", "ODOO_ORDER_CREATED", "SEND_INVOICE"),
"packlink_shipment": (80, "Envio encontrado na Packlink", "SHIPMENT_CREATED", "REVIEW_MANUALLY"),
}
def _process_identity_keys(item: Dict[str, Any]) -> List[Tuple[str, str]]:
"""Return all usable identities for grouping external evidence.
The fiscal customer is still the strongest identity, but real-world data is
uneven: Jasmin usually has NIF while Odoo may only have the partner/company
name. v4.9.17 therefore keeps all identities for the same item and lets the
grouping layer merge evidence by any shared key.
Example: a Jasmin quote with NIF + name and an Odoo sale with only the same
normalized name become one process candidate instead of a quote candidate
with the Odoo sale shown merely as a suggestion.
"""
keys: List[Tuple[str, str]] = []
tax_id = _normalize_tax_id(item.get("customer_tax_id") or "")
if tax_id:
keys.append(("nif", tax_id))
email = _norm_match_value(item.get("customer_email"))
if email:
keys.append(("email", email))
name = _normalize_company_name(item.get("customer_name") or item.get("linked_customer_name"))
if name and len(name) >= 5:
keys.append(("name", name))
return keys
def _process_identity_key(item: Dict[str, Any]) -> Optional[Tuple[str, str]]:
"""Backward-compatible strongest identity for older callers/tests."""
keys = _process_identity_keys(item)
return keys[0] if keys else None
def _best_process_group_key(items: List[Dict[str, Any]]) -> Optional[Tuple[str, str]]:
"""Pick the strongest display key for a merged process group."""
priority = {"nif": 0, "email": 1, "name": 2}
keys: List[Tuple[str, str]] = []
for item in items:
keys.extend(_process_identity_keys(item))
if not keys:
return None
# Deduplicate while preserving deterministic ordering by strength/value.
unique = sorted(set(keys), key=lambda x: (priority.get(x[0], 99), x[1]))
return unique[0]
def _base_process_step(item: Dict[str, Any]) -> Dict[str, Any]:
external_type = str(item.get("external_type") or "external_record")
rank, label, stage, action_code = PROCESS_STEP_BY_EXTERNAL_TYPE.get(
external_type,
(10, "Evidência externa encontrada", STAGE_BY_EXTERNAL_TYPE.get(external_type, "REVIEW"), NEXT_ACTION_BY_EXTERNAL_TYPE.get(external_type, "REVIEW_MANUALLY")),
)
return {
"rank": rank,
"label": label,
"stage": stage,
"action_code": item.get("suggested_action") or action_code,
"item_id": item.get("id"),
"source_system": item.get("source_system"),
"external_type": external_type,
"document_number": item.get("document_number") or item.get("external_id"),
"document_date": item.get("document_date"),
"amount": item.get("amount"),
"currency": item.get("currency") or "EUR",
"title": item.get("title"),
}
def _process_steps_for_item(item: Dict[str, Any]) -> List[Dict[str, Any]]:
base = _base_process_step(item)
if str(item.get("external_type") or "") != "odoo_sale_order":
return [base]
steps = [base]
fulfilment = _odoo_fulfilment_from_item(item)
lines = fulfilment.get("lines") if isinstance(fulfilment.get("lines"), list) else []
if lines:
steps.append({**base, "rank": 62, "label": "Linhas/produtos importados do Odoo", "stage": "ODOO_ORDER_CREATED", "action_code": "SEND_INVOICE", "external_type": "odoo_sale_lines", "products": [line.get("product_name") or line.get("description") for line in lines if line.get("product_name") or line.get("description")]})
outgoing = fulfilment.get("outgoing_pickings") if isinstance(fulfilment.get("outgoing_pickings"), list) else []
done_pickings = [p for p in outgoing if _clean(p.get("state")) == "done"]
ready_pickings = [p for p in outgoing if _clean(p.get("state")) == "assigned"]
if done_pickings:
for picking in done_pickings:
steps.append({**base, "rank": 75, "label": "Entrega Odoo concluída", "stage": "SHIPPED", "action_code": "SEND_INVOICE", "external_type": "odoo_delivery", "document_number": picking.get("name"), "document_date": _clean(picking.get("date_done"))[:10] or base.get("document_date"), "picking_state": picking.get("state")})
elif ready_pickings:
for picking in ready_pickings:
steps.append({**base, "rank": 70, "label": "Picking Odoo reservado — validação física pendente", "stage": "ORDER_PREPARATION", "action_code": "VALIDATE_PHYSICAL_ORDER", "external_type": "odoo_delivery", "document_number": picking.get("name"), "document_date": _clean(picking.get("scheduled_date"))[:10] or base.get("document_date"), "picking_state": picking.get("state")})
if fulfilment.get("invoice_pending"):
steps.append({**base, "rank": 85, "label": "Fatura por emitir", "stage": "SHIPMENT_CREATED" if done_pickings else "ODOO_ORDER_CREATED", "action_code": "SEND_INVOICE", "external_type": "odoo_invoice_pending", "document_number": base.get("document_number")})
return steps
def _process_step(item: Dict[str, Any]) -> Dict[str, Any]:
return _process_steps_for_item(item)[0]
def _operation_document_ref(item: Dict[str, Any]) -> str:
"""Stable document/order reference used to avoid merging two orders.
Customer identity tells us who the company is; this reference tells us which
commercial operation the evidence belongs to. It is deliberately based on
external document/order numbers rather than the customer name.
"""
record = _payload_record(item)
return _clean(
record.get("name")
or item.get("document_number")
or record.get("id")
or item.get("external_id")
)
def _operation_primary_anchor(item: Dict[str, Any]) -> str:
"""Return a strong operation anchor for a specific purchase/process.
Fiscal identity answers "who is the customer". This anchor answers "which
purchase/process is this evidence about". Odoo sales are always anchors,
and Jasmin commercial documents are anchors too so two quotations for the
same fiscal customer do not collapse into one giant opportunity.
"""
external_type = str(item.get("external_type") or "")
ref = _operation_document_ref(item)
anchor_types = {
"odoo_sale_order",
"jasmin_quotation",
"jasmin_proforma",
"jasmin_invoice",
}
if external_type in anchor_types and ref:
return f"{external_type}:{ref}"
return ""
def _date_prefix(value: Any) -> str:
return _clean(value)[:10]
def _date_distance_days(a: Any, b: Any) -> Optional[int]:
a_value = _date_prefix(a)
b_value = _date_prefix(b)
if not a_value or not b_value:
return None
try:
return abs((datetime.fromisoformat(a_value).date() - datetime.fromisoformat(b_value).date()).days)
except Exception:
return None
def _operation_match_score(item: Dict[str, Any], anchor: Dict[str, Any]) -> int:
"""Score whether an evidence item belongs to a specific purchase anchor."""
if item is anchor or item.get("id") == anchor.get("id"):
return 1000
score = 0
item_ref = _operation_document_ref(item)
anchor_ref = _operation_document_ref(anchor)
if item_ref and anchor_ref and item_ref == anchor_ref:
score += 100
item_amount = _money_or_none(item.get("amount"))
anchor_amount = _money_or_none(anchor.get("amount"))
if item_amount is not None and anchor_amount is not None:
try:
item_dec = Decimal(item_amount)
anchor_dec = Decimal(anchor_amount)
if item_dec == anchor_dec:
score += 70
elif anchor_dec and item_dec >= (anchor_dec * Decimal("0.90")) and item_dec <= (anchor_dec * Decimal("1.10")):
score += 35
except (InvalidOperation, ValueError, TypeError):
pass
distance = _date_distance_days(item.get("document_date"), anchor.get("document_date"))
if distance is not None:
if distance == 0:
score += 25
elif distance <= 3:
score += 20
elif distance <= 10:
score += 12
elif distance <= 30:
score += 5
# Odoo fulfilment lines are often the best differentiator when the same
# company places two orders close together. Only count explicit overlap.
item_products = {
_normalize_product_name(line.get("product_name") or line.get("description"))
for line in _odoo_importable_lines(item)
if _normalize_product_name(line.get("product_name") or line.get("description"))
}
anchor_products = {
_normalize_product_name(line.get("product_name") or line.get("description"))
for line in _odoo_importable_lines(anchor)
if _normalize_product_name(line.get("product_name") or line.get("description"))
}
if item_products and anchor_products and item_products & anchor_products:
score += 25
return score
def _operation_anchors_can_merge(left: Dict[str, Any], right: Dict[str, Any]) -> bool:
"""Return true when two purchase anchors are clearly the same process.
Conservative rules:
- Same external document/order reference is a hard match.
- Two Odoo sale orders with different refs are always different purchases.
- Two anchors of the same document class with different refs remain separate.
- Different document classes may merge only with strong amount/date/product
evidence, e.g. quote ORC2026.154 + sale S00279 for the same amount.
"""
left_ref = _operation_document_ref(left)
right_ref = _operation_document_ref(right)
if left_ref and right_ref and left_ref == right_ref:
return True
left_type = str(left.get("external_type") or "")
right_type = str(right.get("external_type") or "")
if left_type == "odoo_sale_order" and right_type == "odoo_sale_order":
return False
if left_type == right_type:
return False
score = _operation_match_score(left, right)
if score >= 90:
return True
# Backward-compatible fiscal-name bridge: when Jasmin gives the NIF/name
# and Odoo only gives the name, a single Odoo sale close to the Jasmin
# commercial document should still reconstruct one process even when the
# Jasmin amount was not imported. Multiple Odoo orders remain separated by
# their own anchors and loose items must still match unambiguously.
if {left_type, right_type} & {"odoo_sale_order"}:
distance = _date_distance_days(left.get("document_date"), right.get("document_date"))
left_amount = _money_or_none(left.get("amount"))
right_amount = _money_or_none(right.get("amount"))
if distance is not None and distance <= 3 and (left_amount is None or right_amount is None):
return True
return False
def _split_items_by_operation_identity(items: List[Dict[str, Any]]) -> List[List[Dict[str, Any]]]:
"""Split one fiscal customer group into separate commercial purchases.
The first pass intentionally merges evidence by fiscal identity
(NIF/email/fiscal name). This second pass prevents "one customer = one
process" mistakes. Every Odoo sale and every Jasmin commercial document is
a purchase anchor; anchors are merged only when there is strong evidence
they are the same purchase. Payments/shipments/manual evidence are assigned
only when the best amount/date/reference/product match is unambiguous.
"""
anchors = [item for item in items if _operation_primary_anchor(item)]
if len(anchors) <= 1:
return [items]
parent = list(range(len(anchors)))
def find(idx: int) -> int:
while parent[idx] != idx:
parent[idx] = parent[parent[idx]]
idx = parent[idx]
return idx
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent[rb] = ra
for i, left in enumerate(anchors):
for j, right in enumerate(anchors[i + 1:], start=i + 1):
if _operation_anchors_can_merge(left, right):
union(i, j)
grouped_anchor_indexes: Dict[int, List[int]] = {}
for idx in range(len(anchors)):
grouped_anchor_indexes.setdefault(find(idx), []).append(idx)
buckets: List[Dict[str, Any]] = []
for indexes in grouped_anchor_indexes.values():
anchor_items = [anchors[idx] for idx in indexes]
# Prefer Odoo sale order as the visible purchase key, then quotation,
# proforma and invoice. This keeps UI labels stable for real orders.
anchor_items.sort(key=lambda item: {
"odoo_sale_order": 0,
"jasmin_quotation": 1,
"jasmin_proforma": 2,
"jasmin_invoice": 3,
}.get(str(item.get("external_type") or ""), 9))
buckets.append({"anchor": anchor_items[0], "items": anchor_items, "key": _operation_primary_anchor(anchor_items[0])})
anchor_ids = {str(anchor.get("id") or id(anchor)) for anchor in anchors}
standalone: List[List[Dict[str, Any]]] = []
for item in items:
if str(item.get("id") or id(item)) in anchor_ids:
continue
scored = sorted(
((_operation_match_score(item, bucket["anchor"]), idx) for idx, bucket in enumerate(buckets)),
key=lambda pair: pair[0],
reverse=True,
)
best_score, best_idx = scored[0]
second_score = scored[1][0] if len(scored) > 1 else -1
if best_score >= 70 and best_score > second_score:
buckets[best_idx]["items"].append(item)
else:
standalone.append([item])
return [bucket["items"] for bucket in buckets] + standalone
def _operation_group_suffix(items: List[Dict[str, Any]]) -> str:
preferred_types = ["odoo_sale_order", "jasmin_quotation", "jasmin_proforma", "jasmin_invoice"]
for preferred_type in preferred_types:
for item in items:
if str(item.get("external_type") or "") != preferred_type:
continue
anchor = _operation_primary_anchor(item)
if anchor:
return anchor
for item in items:
ref = _operation_document_ref(item)
if ref:
return f"document:{ref}"
return ""
def infer_reconciliation_process_state(items: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Infer a conservative process stage/next action from grouped evidence."""
steps = sorted([step for item in items for step in _process_steps_for_item(item)], key=lambda x: (x.get("rank") or 0, str(x.get("document_date") or "")))
if not steps:
return {"stage": "REVIEW", "action_code": "REVIEW_MANUALLY", "label": "Processo por rever"}
strongest = max(steps, key=lambda x: int(x.get("rank") or 0))
stage = strongest.get("stage") or "REVIEW"
action_code = strongest.get("action_code") or "REVIEW_MANUALLY"
# If a sale/order already exists, the important next check is usually
# invoice/payment status, not creating another commercial document.
external_types = {str(item.get("external_type") or "") for item in items}
if "odoo_sale_order" in external_types and "jasmin_invoice" not in external_types:
action_code = "SEND_INVOICE"
if any(step.get("external_type") == "odoo_delivery" for step in steps):
stage = "SHIPPED"
if "payment_proof" in external_types:
action_code = "CONFIRM_PAYMENT"
if "packlink_shipment" in external_types and "jasmin_invoice" not in external_types:
action_code = "SEND_INVOICE"
return {
"stage": stage,
"action_code": action_code,
"label": strongest.get("label") or "Processo reconstruído",
"steps": steps,
}
def _process_amount_values(items: List[Dict[str, Any]]) -> List[Decimal]:
values: List[Decimal] = []
for item in items:
value = _money_or_none(item.get("amount"))
if value is None:
continue
try:
values.append(Decimal(value))
except (InvalidOperation, ValueError, TypeError):
continue
return values
def _candidate_reasons_and_risks(group: Dict[str, Any], items: List[Dict[str, Any]], *, identity_reason: str, confidence_label: str) -> Dict[str, Any]:
"""Explain why a process candidate was proposed and what needs review.
The operator should never have to trust an opaque confidence label. This
helper produces short, UI-ready reasons and risks separated from the final
state/action inference. It is deliberately conservative: customer identity
can be high-confidence while the purchase grouping still needs review.
"""
reasons: List[str] = []
risks: List[str] = []
external_types = {str(item.get("external_type") or "") for item in items}
if identity_reason:
reasons.append(f"cliente fiscal por {identity_reason}")
if group.get("operation_key"):
reasons.append("compra/processo separado por referência documental")
if group.get("split_from_multi_purchase"):
reasons.append("cliente com várias compras detectadas; este cartão foi separado")
risks.append("mesmo cliente fiscal tem outros processos na janela")
if len(items) > 1:
reasons.append(f"{len(items)} evidências agrupadas")
if group.get("suggestions"):
reasons.append("existe sugestão de oportunidade aberta")
amounts = _process_amount_values(items)
amount_conflict = False
amount_conflict_values: List[str] = []
if len(amounts) >= 2:
unique_amounts = sorted(set(amounts))
amount_conflict_values = [str(value.quantize(Decimal("0.01"))) for value in unique_amounts]
min_amount, max_amount = min(unique_amounts), max(unique_amounts)
if len(unique_amounts) == 1:
reasons.append("valor igual entre documentos")
elif min_amount and max_amount <= (min_amount * Decimal("1.10")):
reasons.append("valor aproximado entre documentos")
else:
amount_conflict = True
risks.append("conflito financeiro: valores divergentes entre documentos")
risks.append("bloquear auto-associação financeira até revisão")
dated_items = [item for item in items if item.get("document_date")]
if len(dated_items) >= 2:
distances = [
_date_distance_days(left.get("document_date"), right.get("document_date"))
for pos, left in enumerate(dated_items)
for right in dated_items[pos + 1:]
]
distances = [d for d in distances if d is not None]
if distances:
if min(distances) <= 3:
reasons.append("datas próximas")
if max(distances) > 30:
risks.append("datas afastadas; pode ser histórico ou compra diferente")
if "odoo_sale_order" in external_types and "jasmin_invoice" not in external_types:
risks.append("venda Odoo sem fatura Jasmin associada")
if "jasmin_invoice" in external_types and not ({"odoo_sale_order", "jasmin_quotation", "jasmin_proforma"} & external_types):
risks.append("fatura solta; confirmar se é histórico")
if "payment_proof" in external_types and "jasmin_invoice" not in external_types:
risks.append("comprovativo sem fatura associada")
if not any(_normalize_tax_id(item.get("customer_tax_id") or "") for item in items):
risks.append("cliente sem NIF; validar nome fiscal")
if confidence_label != "alta":
risks.append("confiança de cliente/processo não é alta")
# Deterministic de-duplication with stable order.
reasons = list(dict.fromkeys([r for r in reasons if r]))
risks = list(dict.fromkeys([r for r in risks if r]))
if amount_conflict or (risks and any("valores diferentes" in risk or "valor divergente" in risk or "compra diferente" in risk for risk in risks)):
review_status = "conflict"
elif risks:
review_status = "needs_review"
else:
review_status = "ready"
return {
"reasons": reasons,
"risks": risks,
"review_status": review_status,
"financial_conflict": amount_conflict,
"amount_conflict_values": amount_conflict_values if amount_conflict else [],
}
def _record_reconciliation_decision(
conn: Any,
*,
decision_type: str,
item_ids: Optional[List[str]] = None,
status: Optional[str] = None,
opportunity_id: Optional[str] = None,
customer_id: Optional[str] = None,
note: str = "",
actor: str = "operator",
payload: Optional[Dict[str, Any]] = None,
) -> None:
"""Persist an operator/system decision without failing core actions.
Long-lived installations may receive this table through the additive schema
guard. If a local database is inconsistent, the original link/create/ignore
action should still complete, so failures are logged only.
"""
try:
conn.execute(text("""
INSERT INTO reconciliation_decisions (
decision_type, status, item_ids, opportunity_id, customer_id,
note, actor, payload
) VALUES (
:decision_type, :status, CAST(:item_ids AS TEXT[]),
CAST(:opportunity_id AS UUID), CAST(:customer_id AS UUID),
:note, :actor, CAST(:payload AS JSONB)
)
"""), {
"decision_type": _clean(decision_type) or "unknown",
"status": _clean(status) or None,
"item_ids": [str(x) for x in (item_ids or []) if str(x)],
"opportunity_id": _uuid_or_none(opportunity_id),
"customer_id": _uuid_or_none(customer_id),
"note": _clean(note) or None,
"actor": _clean(actor) or "operator",
"payload": _json(payload or {}),
})
except Exception as exc: # pragma: no cover - production safety guard
logger.warning("failed to record reconciliation decision %s: %s", decision_type, exc)
def list_reconciliation_process_candidates(*, status: str = "open", days: int = 3, limit: int = 20) -> List[Dict[str, Any]]:
"""Return grouped process candidates built from recent open items.
A candidate is shown when at least two pieces of evidence share the same
NIF/email/name, when one item already has a strong open-operation
suggestion, or when one important external item is actionable by itself
(for example an Odoo sale order without ClientFlow).
"""
try:
items = list_reconciliation_items(status=status, limit=300, days=days)
except Exception as exc: # pragma: no cover - production safety guard
logger.warning("failed to list reconciliation items for process candidates: %s", exc)
return []
# Build connected components from every usable identity key. NIF is absolute
# and normalized company fiscal name is also a strong business key in this
# project. This avoids splitting one real process into separate cards when
# Jasmin has NIF + name and Odoo has only the company name.
indexed_items: List[Dict[str, Any]] = []
parent: List[int] = []
key_owner: Dict[str, int] = {}
def find(idx: int) -> int:
while parent[idx] != idx:
parent[idx] = parent[parent[idx]]
idx = parent[idx]
return idx
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent[rb] = ra
for item in items:
keys = _process_identity_keys(item)
if not keys:
continue
idx = len(indexed_items)
indexed_items.append(item)
parent.append(idx)
for key_name, key_value in keys:
key = f"{key_name}:{key_value}"
if key in key_owner:
union(idx, key_owner[key])
else:
key_owner[key] = idx
# Fiscal-name bridge for cross-source reconstruction. Jasmin often carries
# the fiscal NIF + canonical company name, while Odoo may only carry a
# partner/display name with extra location text and no NIF. Exact identity
# keys above do not merge those records when the names are not identical.
# If at least one side has a NIF and the normalized names are a strong
# substring/overlap match, merge them into the same process. The merged
# group will still use the NIF as display key through _best_process_group_key.
name_candidates: List[Tuple[int, str, bool, str]] = []
for idx, item in enumerate(indexed_items):
name_key = _normalize_company_name(item.get("customer_name") or item.get("linked_customer_name"))
if len(name_key) < 5:
continue
has_tax = bool(_normalize_tax_id(item.get("customer_tax_id") or ""))
source = str(item.get("source_system") or "")
name_candidates.append((idx, name_key, has_tax, source))
for pos, (idx_a, name_a, has_tax_a, source_a) in enumerate(name_candidates):
for idx_b, name_b, has_tax_b, source_b in name_candidates[pos + 1:]:
if find(idx_a) == find(idx_b):
continue
if not (has_tax_a or has_tax_b):
continue
# Prefer bridging different sources; same-source fuzzy merges are
# riskier and can accidentally collapse unrelated customer records.
if source_a and source_b and source_a == source_b:
continue
score = _name_match_score(name_a, name_b)
if score >= 35:
union(idx_a, idx_b)
grouped_items: Dict[int, List[Dict[str, Any]]] = {}
for idx, item in enumerate(indexed_items):
grouped_items.setdefault(find(idx), []).append(item)
groups: Dict[str, Dict[str, Any]] = {}
for identity_items in grouped_items.values():
split_groups = _split_items_by_operation_identity(identity_items)
multiple_operation_groups = len(split_groups) > 1
for raw_items in split_groups:
best_key = _best_process_group_key(raw_items)
if not best_key:
continue
key_name, key_value = best_key
identity_group_key = f"{key_name}:{key_value}"
operation_suffix = _operation_group_suffix(raw_items) if multiple_operation_groups else ""
group_key = f"{identity_group_key}|{operation_suffix}" if operation_suffix else identity_group_key
group = {
"process_key": group_key,
"match_key": key_name,
"match_value": key_value,
"operation_key": operation_suffix,
"split_from_multi_purchase": multiple_operation_groups,
"items": [],
"suggestions": [],
}
for item in raw_items:
group["items"].append(item)
for suggestion in item.get("operation_suggestions") or []:
if suggestion.get("opportunity_id") and all(existing.get("opportunity_id") != suggestion.get("opportunity_id") for existing in group["suggestions"]):
group["suggestions"].append(suggestion)
groups[group_key] = group
candidates: List[Dict[str, Any]] = []
for group in groups.values():
group_items = sorted(
group["items"],
key=lambda x: (
str(x.get("document_date") or x.get("updated_at") or ""),
min((int(step.get("rank") or 0) for step in _process_steps_for_item(x)), default=0),
),
)
external_types = {str(item.get("external_type") or "") for item in group_items}
actionable_singleton = bool(external_types & {
"odoo_sale_order",
"jasmin_invoice",
"jasmin_proforma",
"payment_proof",
"packlink_shipment",
})
split_purchase_singleton = bool(group.get("split_from_multi_purchase")) and bool(external_types & {
"jasmin_quotation",
"manual_request",
"external_record",
})
if len(group_items) < 2 and not group.get("suggestions") and not actionable_singleton and not split_purchase_singleton:
continue
state = infer_reconciliation_process_state(group_items)
first = group_items[0]
customer_name = next((item.get("customer_name") for item in group_items if item.get("customer_name")), None) or first.get("linked_customer_name") or "Cliente externo"
customer_email = next((item.get("customer_email") for item in group_items if item.get("customer_email")), None) or ""
customer_tax_id = next((item.get("customer_tax_id") for item in group_items if item.get("customer_tax_id")), None) or ""
total_amount = next((item.get("amount") for item in group_items if item.get("amount") is not None), None)
confidence_label, identity_reason = _process_group_identity_label(str(group.get("match_key") or ""), group_items)
explanation = _candidate_reasons_and_risks(group, group_items, identity_reason=identity_reason, confidence_label=confidence_label)
candidates.append({
"process_key": group["process_key"],
"match_key": group["match_key"],
"match_value": group["match_value"],
"operation_key": group.get("operation_key") or "",
"customer_name": customer_name,
"customer_email": customer_email,
"customer_tax_id": customer_tax_id,
"amount": total_amount,
"currency": first.get("currency") or "EUR",
"item_ids": [str(item.get("id")) for item in group_items if item.get("id")],
"items": group_items,
"timeline": state.get("steps") or [],
"suggested_stage": state.get("stage"),
"suggested_action": state.get("action_code"),
"suggested_label": state.get("label"),
"suggestions": group.get("suggestions") or [],
"confidence": confidence_label,
"identity_reason": identity_reason,
"reasons": explanation.get("reasons") or [],
"risks": explanation.get("risks") or [],
"review_status": explanation.get("review_status") or "needs_review",
"financial_conflict": bool(explanation.get("financial_conflict")),
"amount_conflict_values": explanation.get("amount_conflict_values") or [],
})
candidates.sort(key=lambda x: (0 if x.get("confidence") == "alta" else 1, -len(x.get("items") or []), str(x.get("customer_name") or "")))
return candidates[: max(int(limit or 20), 1)]
def _get_reconciliation_items_by_ids(item_ids: List[str]) -> List[Dict[str, Any]]:
ensure_reconciliation_schema()
cleaned = [str(x).strip() for x in item_ids if str(x).strip()]
if not cleaned:
return []
with engine.begin() as conn:
rows = conn.execute(text("""
SELECT id::text, source_system, external_type, external_id, title, description,
status, priority, suggested_action, confidence, opportunity_id::text,
customer_id::text, customer_name, customer_email, customer_tax_id, document_number,
document_date, amount, currency, payload, resolution_note, created_at,
updated_at, resolved_at
FROM reconciliation_items
WHERE id = ANY(CAST(:ids AS UUID[]))
AND status IN ('open','needs_review','conflict')
ORDER BY document_date NULLS FIRST, created_at
"""), {"ids": cleaned}).mappings().all()
return [dict(row) for row in rows]
def _materialize_current_opportunity_action(opportunity_id: str, *, actor: str) -> Dict[str, Any]:
"""Materialize only the central, post-commit decision for the opportunity."""
try:
from app.opportunity_next_action_service import get_opportunity_next_action
from app.opportunity_action_task_materializer import ensure_pending_task_for_next_action
next_action = get_opportunity_next_action(opportunity_id)
result = ensure_pending_task_for_next_action(
opportunity_id,
next_action,
source="reconciliation_v129",
actor=actor,
)
return {"next_action": next_action, "materialization": result}
except Exception as exc: # pragma: no cover - production safety guard
logger.warning("failed to materialize post-reconciliation action for %s: %s", opportunity_id, exc)
return {"error": str(exc)}
def link_reconciliation_process_to_opportunity(item_ids: List[str], opportunity_id: str, *, actor: str = "operator") -> int:
"""Link several reconciliation items to the same opportunity as one process."""
items = _get_reconciliation_items_by_ids(item_ids)
if not items or not _uuid_or_none(opportunity_id):
return 0
state = infer_reconciliation_process_state(items)
with engine.begin() as conn:
for item in items:
conn.execute(text("""
UPDATE reconciliation_items
SET opportunity_id = CAST(:opportunity_id AS UUID),
status = 'linked',
resolved_at = now(),
updated_at = now(),
payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB)
WHERE id = CAST(:id AS UUID)
"""), {
"id": item.get("id"),
"opportunity_id": opportunity_id,
"payload": _json({"process_reconstruction": {"linked_by": actor, "suggested_stage": state.get("stage"), "suggested_action": state.get("action_code")}}),
})
_upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor)
_upsert_odoo_operation_links_from_item(conn, item, opportunity_id)
_upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id)
_apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor)
for step in _process_steps_for_item(item):
conn.execute(text("""
INSERT INTO opportunity_events (
id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by
) VALUES (
CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID),
'reconciliation_evidence_imported', :action_code, :to_stage,
:note, CAST(:payload AS JSONB), :created_by
)
"""), {
"event_id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"action_code": step.get("action_code") or state.get("action_code"),
"to_stage": step.get("stage") or state.get("stage"),
"note": f"{step.get('label') or 'Evidência externa ligada'}: {step.get('document_number') or item.get('document_number') or item.get('external_id')}",
"payload": _json({"reconciliation_item_id": item.get("id"), "source_system": item.get("source_system"), "external_type": step.get("external_type") or item.get("external_type"), "external_id": item.get("external_id"), "document_number": step.get("document_number") or item.get("document_number"), "document_date": step.get("document_date") or item.get("document_date"), "amount": item.get("amount"), "step": step}),
"created_by": actor,
})
_apply_reconstructed_process_to_opportunity(conn, items, opportunity_id, actor=actor)
_record_reconciliation_decision(
conn,
decision_type="link_process",
item_ids=[str(item.get("id")) for item in items if item.get("id")],
status="linked",
opportunity_id=opportunity_id,
actor=actor,
payload={"suggested_stage": state.get("stage"), "suggested_action": state.get("action_code")},
)
_materialize_current_opportunity_action(opportunity_id, actor=actor)
return len(items)
def create_opportunity_from_reconciliation_process(item_ids: List[str], *, actor: str = "operator") -> Optional[str]:
"""Create one reconstructed opportunity from several external items."""
items = _get_reconciliation_items_by_ids(item_ids)
if not items:
return None
state = infer_reconciliation_process_state(items)
first = items[0]
customer_name = next((item.get("customer_name") for item in items if item.get("customer_name")), None) or "Processo reconstruído"
customer_email = next((item.get("customer_email") for item in items if item.get("customer_email")), None) or None
customer_id = next((item.get("customer_id") for item in items if item.get("customer_id")), None)
amount_value = next((item.get("amount") for item in items if item.get("amount") is not None), None)
currency = next((item.get("currency") for item in items if item.get("currency")), None) or "EUR"
action_code = state.get("action_code") or "REVIEW_MANUALLY"
stage = state.get("stage") or "REVIEW"
evidence = [{
"reconciliation_item_id": item.get("id"),
"source_system": item.get("source_system"),
"external_type": item.get("external_type"),
"external_id": item.get("external_id"),
"document_number": item.get("document_number"),
"document_date": item.get("document_date"),
"amount": item.get("amount"),
} for item in items]
opportunity_id = str(uuid.uuid4())
review_patch = reconstructed_review_metadata_patch(
"required",
actor=actor,
reason="Oportunidade reconstruída a partir de evidências externas.",
blocked_action_code=action_code,
)
metadata = {
"created_from_reconciliation_process": True,
"source_system": "reconciliation",
"evidence": evidence,
"suggested_stage": stage,
"suggested_action": action_code,
"clientflow_record_mode": "reconstructed_invoice_review",
**review_patch,
}
title = f"Processo reconstruído · {customer_name}"
with engine.begin() as conn:
conn.execute(text("""
INSERT INTO opportunities (
id, title, stage, status, local_customer_id, customer_name,
customer_email, product_interest, value_amount, currency,
source_system, source_event_id, last_action_code, metadata
) VALUES (
CAST(:id AS UUID), :title, :stage, 'open', CAST(:customer_id AS UUID),
:customer_name, :customer_email, :product_interest, :value_amount,
:currency, 'reconciliation', :source_event_id, :last_action_code,
CAST(:metadata AS JSONB)
)
"""), {
"id": opportunity_id,
"title": title,
"stage": stage,
"customer_id": _uuid_or_none(customer_id),
"customer_name": customer_name,
"customer_email": customer_email,
"product_interest": "Processo importado/reconstruído",
"value_amount": _money_or_none(amount_value),
"currency": currency,
"source_event_id": first.get("external_id") or first.get("id"),
"last_action_code": action_code,
"metadata": _json(metadata),
})
conn.execute(text("""
INSERT INTO opportunity_events (
id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by
) VALUES (
CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID),
'opportunity_reconstructed_from_reconciliation', :action_code, :to_stage,
:note, CAST(:payload AS JSONB), :created_by
)
"""), {
"event_id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"action_code": action_code,
"to_stage": stage,
"note": "Oportunidade reconstruída a partir de múltiplas evidências externas.",
"payload": _json(metadata),
"created_by": actor,
})
for item in items:
conn.execute(text("""
UPDATE reconciliation_items
SET opportunity_id = CAST(:opportunity_id AS UUID),
status = 'linked',
resolved_at = now(),
updated_at = now(),
resolution_note = 'Ligado por reconstrução de processo',
payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB)
WHERE id = CAST(:id AS UUID)
"""), {"opportunity_id": opportunity_id, "id": item.get("id"), "payload": _json({"process_reconstruction": {"opportunity_id": opportunity_id, "actor": actor}})})
_upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor)
_upsert_odoo_operation_links_from_item(conn, item, opportunity_id)
_upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id)
_apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor)
for step in _process_steps_for_item(item):
conn.execute(text("""
INSERT INTO opportunity_events (
id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by
) VALUES (
CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID),
'reconciliation_evidence_imported', :action_code, :to_stage,
:note, CAST(:payload AS JSONB), :created_by
)
"""), {
"event_id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"action_code": step.get("action_code") or action_code,
"to_stage": step.get("stage") or stage,
"note": f"{step.get('label') or 'Evidência externa importada'}: {step.get('document_number') or item.get('document_number') or item.get('external_id')}",
"payload": _json({"reconciliation_item_id": item.get("id"), "source_system": item.get("source_system"), "external_type": step.get("external_type") or item.get("external_type"), "external_id": item.get("external_id"), "document_number": step.get("document_number") or item.get("document_number"), "document_date": step.get("document_date") or item.get("document_date"), "amount": item.get("amount"), "step": step}),
"created_by": actor,
})
_record_reconciliation_decision(
conn,
decision_type="create_process_opportunity",
item_ids=[str(item.get("id")) for item in items if item.get("id")],
status="linked",
opportunity_id=opportunity_id,
customer_id=_uuid_or_none(customer_id),
note="Oportunidade reconstruída a partir de reconciliação",
actor=actor,
payload={"suggested_stage": stage, "suggested_action": action_code, "evidence_count": len(evidence)},
)
task_id = _create_task_for_opportunity(
opportunity_id=opportunity_id,
action_code="REVIEW_RECONSTRUCTED_PROCESS",
note="Confirmar cliente, documento principal, valor e evidências antes de executar a ação sensível sugerida.",
source_system="reconciliation_process",
source_event_id=opportunity_id,
metadata={**metadata, "blocked_action_code": canonical_action_code(action_code), "review_type": "reconstructed_process"},
)
if task_id:
with engine.begin() as conn:
conn.execute(text("""
UPDATE opportunities
SET last_task_id = CAST(:task_id AS UUID), updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
"""), {"task_id": task_id, "opportunity_id": opportunity_id})
return opportunity_id
def set_reconciliation_status(item_id: str, *, status: str, note: str = "", actor: str = "operator") -> None:
ensure_reconciliation_schema()
with engine.begin() as conn:
conn.execute(text("""
UPDATE reconciliation_items
SET status = :status,
resolution_note = COALESCE(NULLIF(:note, ''), resolution_note),
resolved_at = CASE WHEN :status IN ('resolved','ignored','linked','historical') THEN now() ELSE resolved_at END,
updated_at = now(),
payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB)
WHERE id = CAST(:id AS UUID)
"""), {
"id": item_id,
"status": status,
"note": note,
"payload": _json({"last_action": {"actor": actor, "status": status}}),
})
_record_reconciliation_decision(
conn,
decision_type="status",
item_ids=[item_id],
status=status,
note=note,
actor=actor,
payload={"status": status},
)
def link_reconciliation_to_opportunity(item_id: str, opportunity_id: str, *, actor: str = "operator") -> None:
# Treat a single item link as a one-item process link. This ensures Odoo
# sales/fulfilment evidence updates the opportunity pipeline instead of
# creating only a loose reference.
if link_reconciliation_process_to_opportunity([item_id], opportunity_id, actor=actor):
return
ensure_reconciliation_schema()
with engine.begin() as conn:
conn.execute(text("""
UPDATE reconciliation_items
SET opportunity_id = CAST(:opportunity_id AS UUID),
status = 'linked',
resolved_at = now(),
updated_at = now(),
payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB)
WHERE id = CAST(:id AS UUID)
"""), {
"id": item_id,
"opportunity_id": opportunity_id,
"payload": _json({"linked_by": actor}),
})
conn.execute(text("""
INSERT INTO opportunity_events (
id, opportunity_id, event_type, note, payload, created_by
) VALUES (
CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID),
'reconciliation_item_linked', :note, CAST(:payload AS JSONB), :created_by
)
"""), {
"event_id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"note": "Item de reconciliação ligado à oportunidade.",
"payload": _json({"reconciliation_item_id": item_id}),
"created_by": actor,
})
_record_reconciliation_decision(
conn,
decision_type="link_item",
item_ids=[item_id],
status="linked",
opportunity_id=opportunity_id,
actor=actor,
)
def _create_task_for_opportunity(
*,
opportunity_id: str,
action_code: str,
note: str,
source_system: str,
source_event_id: Optional[str],
metadata: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
config = get_action_config(action_code)
route = config.get("route") or "rever"
action = config.get("action") or action_code
action_required = bool(config.get("action_required", True))
safe_to_post = bool(config.get("safe_to_post", False))
status = "pending" if action_required else "skipped"
idempotency_key = f"task:{source_system}:opportunity:{opportunity_id}:{source_event_id or action_code}:{action_code}"
with engine.begin() as conn:
row = conn.execute(text("""
INSERT INTO tasks (
opportunity_id, action_code, route, action, note, action_required,
safe_to_post, status, source_system, source_event_id, idempotency_key,
metadata, priority
) VALUES (
CAST(:opportunity_id AS UUID), :action_code, :route, :action, :note,
:action_required, :safe_to_post, :status, :source_system, :source_event_id,
:idempotency_key, CAST(:metadata AS JSONB), :priority
)
ON CONFLICT (idempotency_key) DO NOTHING
RETURNING id::text
"""), {
"opportunity_id": opportunity_id,
"action_code": action_code,
"route": route,
"action": action,
"note": note,
"action_required": action_required,
"safe_to_post": safe_to_post,
"status": status,
"source_system": source_system,
"source_event_id": source_event_id,
"idempotency_key": idempotency_key,
"metadata": _json(metadata or {}),
"priority": _priority_for_action(action_code),
}).fetchone()
return row[0] if row else None
def create_opportunity_from_reconciliation(item_id: str, *, actor: str = "operator") -> Optional[str]:
"""Promote a reconciliation candidate into a ClientFlow opportunity.
This is a deliberate operator action. It does not auto-confirm payment or
close the opportunity; it creates the process container and a next task.
"""
ensure_reconciliation_schema()
item = get_reconciliation_item(item_id)
if not item:
return None
external_type = str(item.get("external_type") or "external_record")
stage = STAGE_BY_EXTERNAL_TYPE.get(external_type, "REVIEW")
action_code = item.get("suggested_action") or NEXT_ACTION_BY_EXTERNAL_TYPE.get(external_type, "REVIEW_MANUALLY")
title = item.get("title") or item.get("customer_name") or "Oportunidade importada"
customer_name = item.get("customer_name") or title
metadata = {
"created_from_reconciliation_item_id": item_id,
"source_system": item.get("source_system"),
"external_type": external_type,
"external_id": item.get("external_id"),
"document_number": item.get("document_number"),
}
opportunity_id = str(uuid.uuid4())
with engine.begin() as conn:
conn.execute(text("""
INSERT INTO opportunities (
id, title, stage, status, local_customer_id, customer_name,
customer_email, product_interest, value_amount, currency,
source_system, source_event_id, last_action_code, metadata
) VALUES (
CAST(:id AS UUID), :title, :stage, 'open', CAST(:customer_id AS UUID),
:customer_name, :customer_email, :product_interest, :value_amount,
:currency, :source_system, :source_event_id, :last_action_code,
CAST(:metadata AS JSONB)
)
"""), {
"id": opportunity_id,
"title": title,
"stage": stage,
"customer_id": _uuid_or_none(item.get("customer_id")),
"customer_name": customer_name,
"customer_email": item.get("customer_email"),
"product_interest": (item.get("payload") or {}).get("product_interest") if isinstance(item.get("payload"), dict) else None,
"value_amount": _money_or_none(item.get("amount")),
"currency": item.get("currency") or "EUR",
"source_system": item.get("source_system") or "reconciliation",
"source_event_id": item.get("external_id"),
"last_action_code": action_code,
"metadata": _json(metadata),
})
conn.execute(text("""
INSERT INTO opportunity_events (
id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by
) VALUES (
CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID),
'opportunity_created_from_reconciliation', :action_code, :to_stage,
:note, CAST(:payload AS JSONB), :created_by
)
"""), {
"event_id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"action_code": action_code,
"to_stage": stage,
"note": "Oportunidade criada a partir de informação externa reconciliada.",
"payload": _json(metadata),
"created_by": actor,
})
conn.execute(text("""
UPDATE reconciliation_items
SET opportunity_id = CAST(:opportunity_id AS UUID), status = 'linked',
resolved_at = now(), updated_at = now(), resolution_note = :note
WHERE id = CAST(:item_id AS UUID)
"""), {
"opportunity_id": opportunity_id,
"item_id": item_id,
"note": "Criada oportunidade a partir deste item.",
})
_upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor)
_upsert_odoo_operation_links_from_item(conn, item, opportunity_id)
_upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id)
_apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor)
_record_reconciliation_decision(
conn,
decision_type="create_item_opportunity",
item_ids=[item_id],
status="linked",
opportunity_id=opportunity_id,
customer_id=_uuid_or_none(item.get("customer_id")),
note="Criada oportunidade a partir deste item.",
actor=actor,
payload=metadata,
)
task_id = _create_task_for_opportunity(
opportunity_id=opportunity_id,
action_code=action_code,
note=item.get("description") or f"Continuar processo importado de {item.get('source_system') or 'fonte externa'}.",
source_system="reconciliation",
source_event_id=item_id,
metadata=metadata,
)
if task_id:
with engine.begin() as conn:
conn.execute(text("""
UPDATE opportunities
SET last_task_id = CAST(:task_id AS UUID), updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
"""), {"task_id": task_id, "opportunity_id": opportunity_id})
return opportunity_id
def create_external_request(
*,
source_channel: str,
customer_name: str,
customer_email: str = "",
customer_phone: str = "",
product_interest: str = "",
request_text: str = "",
action_code: str = "SEND_QUOTE",
actor: str = "operator",
) -> Dict[str, Any]:
"""Create an opportunity and task for WhatsApp/phone/direct-email intake."""
ensure_reconciliation_schema()
action_code = str(action_code or "SEND_QUOTE").upper()
config = get_action_config(action_code)
opportunity_id = str(uuid.uuid4())
title_parts = [p for p in [_clean(customer_name), _clean(product_interest)] if p]
title = " · ".join(title_parts) or "Pedido externo"
stage = STAGE_BY_EXTERNAL_TYPE.get("manual_request", "QUOTE_REQUESTED")
if action_code == "SEND_INFO":
stage = "INFO_REQUESTED"
elif action_code == "SEND_INVOICE":
stage = "INVOICE_REQUESTED"
elif action_code == "SEND_PROFORMA":
stage = "PROFORMA_REQUESTED"
metadata = {
"manual_intake": True,
"source_channel": source_channel,
"request_text": request_text,
"created_by": actor,
}
with engine.begin() as conn:
conn.execute(text("""
INSERT INTO opportunities (
id, title, stage, status, customer_name, customer_email, customer_phone,
product_interest, source_system, last_action_code, metadata
) VALUES (
CAST(:id AS UUID), :title, :stage, 'open', :customer_name,
:customer_email, :customer_phone, :product_interest, :source_system,
:action_code, CAST(:metadata AS JSONB)
)
"""), {
"id": opportunity_id,
"title": title,
"stage": stage,
"customer_name": _clean(customer_name) or "Contacto externo",
"customer_email": _clean(customer_email),
"customer_phone": _clean(customer_phone),
"product_interest": _clean(product_interest),
"source_system": _clean(source_channel) or "manual",
"action_code": action_code,
"metadata": _json(metadata),
})
conn.execute(text("""
INSERT INTO opportunity_events (
id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by
) VALUES (
CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), 'manual_external_intake',
:action_code, :stage, :note, CAST(:payload AS JSONB), :created_by
)
"""), {
"event_id": str(uuid.uuid4()),
"opportunity_id": opportunity_id,
"action_code": action_code,
"stage": stage,
"note": f"Pedido registado manualmente via {_clean(source_channel) or 'canal externo'}.",
"payload": _json(metadata),
"created_by": actor,
})
task_id = _create_task_for_opportunity(
opportunity_id=opportunity_id,
action_code=action_code,
note=_clean(request_text) or config.get("action") or "Continuar pedido externo.",
source_system=_clean(source_channel) or "manual",
source_event_id=f"manual:{opportunity_id}",
metadata=metadata,
)
if task_id:
with engine.begin() as conn:
conn.execute(text("""
UPDATE opportunities
SET last_task_id = CAST(:task_id AS UUID), updated_at = now()
WHERE id = CAST(:opportunity_id AS UUID)
"""), {"task_id": task_id, "opportunity_id": opportunity_id})
return {"opportunity_id": opportunity_id, "task_id": task_id}
def create_payment_proof(
*,
opportunity_id: Optional[str] = None,
customer_id: Optional[str] = None,
source_system: str = "manual",
source_ref: str = "",
filename: str = "",
file_url: str = "",
amount: Any = None,
currency: str = "EUR",
proof_date: str = "",
note: str = "",
payload: Optional[Dict[str, Any]] = None,
actor: str = "operator",
) -> Dict[str, Any]:
"""Store proof as evidence and create a CONFIRM_PAYMENT task if linked.
A payment proof is never a payment confirmation. It only opens or supports
a finance validation task.
"""
ensure_reconciliation_schema()
amount_value = _money_or_none(amount)
with engine.begin() as conn:
row = conn.execute(text("""
INSERT INTO payment_proofs (
opportunity_id, customer_id, source_system, source_ref, filename,
file_url, amount, currency, proof_date, status, note, payload
) VALUES (
CAST(:opportunity_id AS UUID), CAST(:customer_id AS UUID), :source_system,
:source_ref, :filename, :file_url, :amount, :currency,
CAST(:proof_date AS DATE), 'pending_validation', :note, CAST(:payload AS JSONB)
)
RETURNING id::text, opportunity_id::text, customer_id::text, source_system,
source_ref, filename, file_url, amount, currency, proof_date,
status, note, payload, created_at, updated_at
"""), {
"opportunity_id": _uuid_or_none(opportunity_id),
"customer_id": _uuid_or_none(customer_id),
"source_system": _clean(source_system) or "manual",
"source_ref": _clean(source_ref),
"filename": _clean(filename),
"file_url": _clean(file_url),
"amount": amount_value,
"currency": _clean(currency) or "EUR",
"proof_date": _clean(proof_date) or None,
"note": _clean(note),
"payload": _json(payload or {"created_by": actor}),
}).mappings().first()
proof = dict(row or {})
proof_id = proof.get("id")
if opportunity_id:
task_id = _create_task_for_opportunity(
opportunity_id=opportunity_id,
action_code="CONFIRM_PAYMENT",
note=f"Comprovativo de pagamento recebido. Validar no banco/Jasmin antes de confirmar. {note}".strip(),
source_system="payment_proof",
source_event_id=proof_id,
metadata={"payment_proof_id": proof_id, "amount": amount_value, "source_system": source_system},
)
proof["task_id"] = task_id
else:
item = upsert_reconciliation_item(
source_system=_clean(source_system) or "manual",
external_type="payment_proof",
external_id=proof_id,
title="Comprovativo de pagamento por associar",
description="Comprovativo recebido, mas ainda sem oportunidade associada. Não confirma pagamento.",
priority="alta",
suggested_action="CONFIRM_PAYMENT",
amount=amount_value,
currency=currency,
payload={"payment_proof_id": proof_id, "filename": filename, "note": note},
)
proof["reconciliation_item_id"] = item.get("id")
return proof
def sync_local_documents_without_opportunity(limit: int = 200) -> Dict[str, int]:
"""Create reconciliation candidates for local Jasmin documents not linked to opportunities."""
ensure_reconciliation_schema()
with engine.begin() as conn:
rows = conn.execute(text("""
SELECT cd.id::text, cd.system, cd.document_kind, cd.external_id, cd.document_number,
cd.customer_id::text, c.name AS customer_name, c.email AS customer_email, c.tax_id AS customer_tax_id,
cd.status, cd.total_amount, cd.amount, cd.currency, cd.document_date,
cd.payload
FROM commercial_documents cd
LEFT JOIN customers c ON c.id = cd.customer_id
WHERE cd.opportunity_id IS NULL
AND COALESCE(cd.system, 'jasmin') = 'jasmin'
AND COALESCE(cd.status, '') NOT IN ('ignored','cancelled','superseded')
ORDER BY cd.updated_at DESC, cd.created_at DESC
LIMIT :limit
"""), {"limit": int(limit)}).mappings().all()
created = 0
for row in rows:
data = dict(row)
kind = str(data.get("document_kind") or "document")
external_type = "jasmin_invoice" if "invoice" in kind else "jasmin_quotation"
document_number = data.get("document_number") or data.get("external_id") or data.get("id")
title = f"Documento Jasmin sem oportunidade · {document_number}"
item = upsert_reconciliation_item(
source_system="jasmin",
external_type=external_type,
external_id=data.get("external_id") or data.get("id"),
title=title,
description="Documento Jasmin existente sem ligação a oportunidade ClientFlow.",
priority="normal" if external_type == "jasmin_quotation" else "alta",
suggested_action=NEXT_ACTION_BY_EXTERNAL_TYPE.get(external_type),
customer_id=data.get("customer_id"),
customer_name=data.get("customer_name"),
customer_email=data.get("customer_email"),
customer_tax_id=data.get("customer_tax_id"),
document_number=document_number,
document_date=data.get("document_date"),
amount=data.get("total_amount") or data.get("amount"),
currency=data.get("currency") or "EUR",
payload={"commercial_document_id": data.get("id"), "document_kind": kind, "status": data.get("status")},
idempotency_key=f"reconcile:commercial_document:{data.get('id')}",
)
if item:
created += 1
return {"seen": len(rows), "created_or_updated": created}
Reference in New Issue View Git Blame Copy Permalink