admin/clientflow_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement document reconciliation v2

Browse Source
This commit is contained in:
plx
2026-08-05 23:55:33 +00:00
parent 4a60607503
commit 29382ec6c1
25 changed files with 2082 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
app/commercial_service.py
View File

@@ -158,104 +158,12 @@ def ensure_commercial_schema() -> None:
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_kind ON commercial_documents(document_kind, status)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_role ON commercial_documents(opportunity_id, document_kind, role, is_primary)"))
# Hotfix v4927.1: em bases já usadas podem existir vários documentos
# Jasmin/Odoo ligados à mesma oportunidade e ao mesmo tipo. Ao adicionar
# role/is_primary com DEFAULT current/TRUE, todos passariam a ser
# “atuais”, fazendo o índice único falhar no arranque e deixando o
# serviço indisponível atrás do nginx. Antes de criar o índice, mantemos
# só o documento mais recente como primário e movemos os restantes para
# histórico.
conn.execute(text("""
WITH ranked AS (
SELECT
id,
ROW_NUMBER() OVER (
PARTITION BY opportunity_id, system, document_kind, COALESCE(role, 'current')
ORDER BY COALESCE(updated_at, created_at) DESC, created_at DESC, id DESC
) AS rn
FROM commercial_documents
WHERE opportunity_id IS NOT NULL
AND COALESCE(is_primary, TRUE) = TRUE
AND COALESCE(role, 'current') IN ('current', 'accepted')
)
UPDATE commercial_documents AS d
SET role = CASE
WHEN COALESCE(d.role, 'current') IN ('current', 'accepted') THEN 'historical'
ELSE COALESCE(d.role, 'historical')
END,
is_primary = FALSE,
is_active = FALSE,
updated_at = now()
FROM ranked AS r
WHERE d.id = r.id
AND r.rn > 1
"""))
# v4927.2: não criar índice UNIQUE durante o arranque.
# Em produção podem existir duplicados históricos que ainda não foram
# reconciliados; se o índice único falhar, toda a app fica indisponível
# e o nginx devolve 502. A regra de “um principal por fase” fica
# aplicada pela normalização acima e pelos serviços que promovem um
# documento a atual. O índice de apoio é não único e seguro em bases
# reais com dados legados.
# v2: startup must never classify or rewrite document relationships.
# The versioned migration creates the authoritative partial unique
# constraint; legacy fields below are dual-write compatibility only.
conn.execute(text("DROP INDEX IF EXISTS ux_commercial_documents_primary_role"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_primary_role ON commercial_documents(opportunity_id, system, document_kind, role) WHERE opportunity_id IS NOT NULL AND COALESCE(is_primary, TRUE) = TRUE AND COALESCE(role, 'current') IN ('current', 'accepted')"))
# v4928.1: migração leve de registos antigos/reconstruídos.
# Não apaga documentos nem altera fases. Apenas evita que orçamentos e
# pró-formas legados continuem a aparecer como documento principal quando
# já existe fatura Jasmin atual/aceite para a mesma oportunidade.
conn.execute(text("""
WITH primary_invoice AS (
SELECT DISTINCT ON (opportunity_id, system)
id, opportunity_id, system, COALESCE(document_date, created_at::date) AS invoice_date, created_at
FROM commercial_documents
WHERE opportunity_id IS NOT NULL
AND system = 'jasmin'
AND document_kind = 'invoice'
AND COALESCE(is_primary, TRUE) = TRUE
AND COALESCE(role, 'current') IN ('current', 'accepted')
ORDER BY opportunity_id, system, COALESCE(document_date, created_at::date) DESC, created_at DESC
)
UPDATE commercial_documents AS d
SET role = 'historical',
is_primary = FALSE,
is_active = FALSE,
payload = COALESCE(d.payload, '{}'::jsonb) || jsonb_build_object('legacy_reason', 'superseded_by_invoice_v4928_1'),
updated_at = now()
FROM primary_invoice AS inv
WHERE d.opportunity_id = inv.opportunity_id
AND d.system = inv.system
AND d.document_kind IN ('quotation', 'proforma')
AND COALESCE(d.role, 'current') IN ('current', 'accepted')
AND COALESCE(d.is_primary, TRUE) = TRUE
AND COALESCE(d.document_date, d.created_at::date) <= inv.invoice_date
"""))
# Marca oportunidades reconstruídas com fatura e sem task pendente para a UI
# poder explicar que são processos antigos/auditoria, não um funil limpo.
conn.execute(text("""
UPDATE opportunities AS o
SET metadata = COALESCE(o.metadata, '{}'::jsonb) || jsonb_build_object(
'clientflow_record_mode', COALESCE(o.metadata->>'clientflow_record_mode', 'reconstructed_invoice_review'),
'clientflow_legacy_migrated_by', 'v4928_1'
),
updated_at = now()
WHERE EXISTS (
SELECT 1 FROM commercial_documents d
WHERE d.opportunity_id = o.id
AND d.system = 'jasmin'
AND d.document_kind = 'invoice'
AND COALESCE(d.role, 'current') IN ('current', 'accepted')
AND COALESCE(d.is_primary, TRUE) = TRUE
)
AND NOT EXISTS (
SELECT 1 FROM tasks t
WHERE t.opportunity_id = o.id
AND t.status = 'pending'
)
AND COALESCE(o.metadata->>'clientflow_record_mode', '') = ''
"""))
conn.execute(text("""
CREATE TABLE IF NOT EXISTS commercial_document_lines (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -274,6 +182,9 @@ def ensure_commercial_schema() -> None:
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""))
# Reconciliation v2 columns belong exclusively to migration 007.
# Startup must leave a pre-007 database unchanged so the migration can
# add, backfill and constrain the canonical identifier atomically.
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_document_lines_document ON commercial_document_lines(document_id)"))
conn.execute(text("""
@@ -500,8 +411,13 @@ def create_commercial_document(
if not document_kind:
raise ValueError("document_kind é obrigatório")
with engine.begin() as conn:
from app.document_reconciliation_service import (
document_reconciliation_v2_available,
register_created_document,
)
v2_available = document_reconciliation_v2_available(conn)
version_number = _next_document_version(conn, opportunity_id, document_kind)
if document_kind == "quotation" and opportunity_id:
if document_kind == "quotation" and opportunity_id and not v2_available:
conn.execute(text("""
UPDATE commercial_documents
SET status = CASE WHEN status IN ('created','sent','draft') THEN 'superseded' ELSE status END,
@@ -553,19 +469,46 @@ def create_commercial_document(
"due_date": due_date,
"payload": _json(payload),
}).mappings().first()
if row and opportunity_id and v2_available:
link = register_created_document(
conn, opportunity_id, str(row["id"]), document_kind,
status=status, actor="commercial_service.create_commercial_document",
)
if link:
row = dict(row) | {
"document_id": str(row["id"]),
"link_id": str(link["id"]),
"relationship": link["relationship"],
}
return dict(row or {})
def add_document_lines(document_id: str, lines: Iterable[Dict[str, Any]]) -> None:
ensure_commercial_schema()
with engine.begin() as conn:
# Migration 007 is additive. Select the statement once per batch so
# the generic writer works both before and after the new NOT NULL
# canonical column is introduced.
has_canonical_id = bool(conn.execute(text("""
SELECT EXISTS (
SELECT 1
FROM pg_attribute
WHERE attrelid = to_regclass('commercial_document_lines')
AND attname = 'commercial_document_id'
AND attnum > 0
AND NOT attisdropped
)
""")).scalar())
for idx, line in enumerate(lines):
conn.execute(text("""
columns = ("document_id, commercial_document_id, " if has_canonical_id else "document_id, ")
values = ("CAST(:document_id AS UUID), CAST(:document_id AS UUID), " if has_canonical_id
else "CAST(:document_id AS UUID), ")
conn.execute(text(f"""
INSERT INTO commercial_document_lines (
document_id, opportunity_item_id, line_index, local_product_id, jasmin_sales_item,
{columns}opportunity_item_id, line_index, local_product_id, jasmin_sales_item,
description, quantity, unit, unit_price, tax_schema, total_amount, payload
) VALUES (
CAST(:document_id AS UUID), CAST(:opportunity_item_id AS UUID), :line_index,
{values}CAST(:opportunity_item_id AS UUID), :line_index,
CAST(:local_product_id AS UUID), :jasmin_sales_item, :description, :quantity,
:unit, :unit_price, :tax_schema, :total_amount, CAST(:payload AS JSONB)
)
@@ -587,6 +530,15 @@ def add_document_lines(document_id: str, lines: Iterable[Dict[str, Any]]) -> Non
def list_commercial_documents(opportunity_id: Optional[str] = None, customer_id: Optional[str] = None, limit: int = 100) -> List[Dict[str, Any]]:
ensure_commercial_schema()
if opportunity_id:
from app.document_reconciliation_service import resolve_document_links
# Canonical resolver owns the only group-complete rollout fallback.
rows = resolve_document_links(opportunity_id)
if customer_id:
rows = [row for row in rows if str(row.get("customer_id") or "") == str(customer_id)]
return rows[:int(limit)]
# Legitimate legacy inventory: without an opportunity scope this endpoint
# lists document records for customer/history, not effective relationships.
filters = []
params: Dict[str, Any] = {"limit": int(limit)}
if opportunity_id:
@@ -602,15 +554,17 @@ def list_commercial_documents(opportunity_id: Optional[str] = None, customer_id:
cd.document_kind, cd.external_id, cd.external_url, cd.company, cd.document_type,
cd.serie, cd.series_number, cd.document_number, cd.customer_party_key,
cd.status, cd.amount, cd.tax_amount, cd.total_amount, cd.currency,
cd.parent_document_id::text, cd.version_number, cd.role, cd.is_primary, cd.is_active,
cd.parent_document_id::text, cd.version_number,
cd.role, cd.is_primary, cd.is_active,
NULL::text AS relationship, NULL::boolean AS is_manual,
NULL::text AS decision_reason, NULL::text AS decided_by, NULL::timestamptz AS decided_at,
cd.document_date, cd.due_date, cd.payload, cd.created_at, cd.updated_at,
c.name AS customer_name, c.tax_id AS customer_tax_id
FROM commercial_documents cd
LEFT JOIN customers c ON c.id = cd.customer_id
{where}
ORDER BY
CASE COALESCE(cd.role, 'current') WHEN 'current' THEN 0 WHEN 'accepted' THEN 1 WHEN 'related' THEN 2 WHEN 'historical' THEN 3 ELSE 4 END,
COALESCE(cd.is_primary, FALSE) DESC,
cd.is_primary DESC,
cd.created_at DESC
LIMIT :limit
"""), params).mappings().all()
@@ -620,13 +574,20 @@ def list_commercial_documents(opportunity_id: Optional[str] = None, customer_id:
def get_commercial_document(document_id: str) -> Optional[Dict[str, Any]]:
ensure_commercial_schema()
from app.document_reconciliation_service import resolve_document_opportunity, resolve_document_links
opportunity_id = resolve_document_opportunity(document_id)
if opportunity_id:
return next((row for row in resolve_document_links(opportunity_id, include_ended=True)
if str(row.get("document_id")) == str(document_id)), None)
# Legitimate orphan/history lookup: there is no relationship to resolve.
with engine.begin() as conn:
row = conn.execute(text("""
SELECT cd.id::text, cd.customer_id::text, cd.opportunity_id::text, cd.system,
cd.document_kind, cd.external_id, cd.external_url, cd.company, cd.document_type,
cd.serie, cd.series_number, cd.document_number, cd.customer_party_key,
cd.status, cd.amount, cd.tax_amount, cd.total_amount, cd.currency,
cd.parent_document_id::text, cd.version_number, cd.role, cd.is_primary, cd.is_active,
cd.parent_document_id::text, cd.version_number,
cd.role, cd.is_primary, cd.is_active,
cd.document_date, cd.due_date, cd.payload, cd.created_at, cd.updated_at,
c.name AS customer_name, c.tax_id AS customer_tax_id
FROM commercial_documents cd
@@ -638,20 +599,8 @@ def get_commercial_document(document_id: str) -> Optional[Dict[str, Any]]:
def get_latest_active_quotation(opportunity_id: str) -> Optional[Dict[str, Any]]:
ensure_commercial_schema()
with engine.begin() as conn:
row = conn.execute(text("""
SELECT id::text, customer_id::text, opportunity_id::text, system, document_kind,
external_id, external_url, company, document_type, serie, series_number,
document_number, customer_party_key, status, amount, total_amount, currency,
parent_document_id::text, version_number, is_active, payload, created_at, updated_at
FROM commercial_documents
WHERE opportunity_id = CAST(:opportunity_id AS UUID)
AND document_kind = 'quotation'
AND status NOT IN ('cancelled','failed')
ORDER BY is_active DESC, created_at DESC
LIMIT 1
"""), {"opportunity_id": opportunity_id}).mappings().first()
return dict(row) if row else None
from app.document_reconciliation_service import get_primary_document
return get_primary_document(opportunity_id, "quotation")
def find_invoice_for_parent(parent_document_id: str) -> Optional[Dict[str, Any]]: