439 lines
19 KiB
Python
Executable File
439 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Audit Jasmin invoices created by converting quotations.
|
|
|
|
This script is diagnostic only. It does not modify ClientFlow or Jasmin data.
|
|
It compares local commercial_documents, remote Jasmin document details, and the
|
|
PDF returned by the Jasmin print endpoint for converted invoices.
|
|
|
|
Main goal: detect why invoices converted from quotations may have unexpected
|
|
PDF format/size/layout.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import engine
|
|
from app.jasmin_client import JasminClient
|
|
from app.commercial_service import update_commercial_document_details
|
|
|
|
A4_PORTRAIT_PT = (595.0, 842.0)
|
|
A4_LANDSCAPE_PT = (842.0, 595.0)
|
|
POINT_TO_MM = 25.4 / 72.0
|
|
|
|
|
|
def _s(value: Any) -> str:
|
|
return str(value or "").strip()
|
|
|
|
|
|
def _json_default(value: Any) -> str:
|
|
try:
|
|
return value.isoformat() # type: ignore[attr-defined]
|
|
except Exception:
|
|
return str(value)
|
|
|
|
|
|
def _payload(value: Any) -> dict[str, Any]:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if not value:
|
|
return {}
|
|
try:
|
|
return json.loads(value)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _dig(data: Any, *path: str) -> Any:
|
|
cur = data
|
|
for key in path:
|
|
if isinstance(cur, dict):
|
|
cur = cur.get(key)
|
|
else:
|
|
return None
|
|
return cur
|
|
|
|
|
|
def _first(data: dict[str, Any], keys: Iterable[str]) -> Any:
|
|
for key in keys:
|
|
if data.get(key) not in (None, ""):
|
|
return data.get(key)
|
|
return None
|
|
|
|
|
|
def _money(value: Any) -> str:
|
|
if isinstance(value, dict):
|
|
for k in ("amount", "value", "baseAmount", "reportingAmount"):
|
|
if value.get(k) is not None:
|
|
return _s(value.get(k))
|
|
return _s(value)
|
|
|
|
|
|
def _selected_jasmin_fields(raw: Any) -> dict[str, Any]:
|
|
if not isinstance(raw, dict):
|
|
return {"raw_type": type(raw).__name__, "raw_preview": _s(raw)[:300]}
|
|
return {
|
|
"id": _first(raw, ["id", "key"]),
|
|
"naturalKey": raw.get("naturalKey"),
|
|
"documentNumber": raw.get("documentNumber"),
|
|
"documentType": _first(raw, ["documentType", "documentTypeKey"]),
|
|
"serie": _first(raw, ["serie", "serieKey"]),
|
|
"seriesNumber": _first(raw, ["seriesNumber", "number"]),
|
|
"company": _first(raw, ["company", "companyKey"]),
|
|
"documentDate": raw.get("documentDate"),
|
|
"postingDate": raw.get("postingDate"),
|
|
"currency": _first(raw, ["currency", "currencyKey"]),
|
|
"totalAmount": _money(raw.get("totalAmount")),
|
|
"payableAmount": _money(raw.get("payableAmount")),
|
|
"taxExclusiveAmount": _money(raw.get("taxExclusiveAmount")),
|
|
"buyerCustomerParty": _first(raw, ["buyerCustomerParty", "buyerCustomerPartyKey"]),
|
|
"buyerCustomerPartyName": raw.get("buyerCustomerPartyName"),
|
|
"paymentTerm": _first(raw, ["paymentTerm", "paymentTermKey"]),
|
|
"paymentMethod": _first(raw, ["paymentMethod", "paymentMethodKey"]),
|
|
"printLayout": _first(raw, ["printLayout", "report", "reportName", "reportKey", "documentReport", "printTemplate"]),
|
|
"schema_hint_keys": sorted([k for k in raw.keys() if re.search(r"print|report|layout|template|format|paper|serie|type", k, re.I)])[:50],
|
|
"line_count": len(raw.get("documentLines") or raw.get("lines") or []) if isinstance(raw.get("documentLines") or raw.get("lines") or [], list) else None,
|
|
}
|
|
|
|
|
|
def _extract_pdf_boxes(data: bytes) -> list[dict[str, Any]]:
|
|
# Works for the normal uncompressed page dictionaries returned by Jasmin.
|
|
# It is intentionally dependency-free for production servers.
|
|
latin = data.decode("latin-1", errors="ignore")
|
|
boxes: list[dict[str, Any]] = []
|
|
pattern = re.compile(
|
|
r"/(MediaBox|CropBox)\s*\[\s*([-+0-9.]+)\s+([-+0-9.]+)\s+([-+0-9.]+)\s+([-+0-9.]+)\s*\]",
|
|
re.I,
|
|
)
|
|
seen: set[tuple[str, float, float, float, float]] = set()
|
|
for m in pattern.finditer(latin):
|
|
kind = m.group(1)
|
|
x0, y0, x1, y1 = (float(m.group(i)) for i in range(2, 6))
|
|
key = (kind.lower(), x0, y0, x1, y1)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
w = abs(x1 - x0)
|
|
h = abs(y1 - y0)
|
|
boxes.append({
|
|
"kind": kind,
|
|
"width_pt": round(w, 2),
|
|
"height_pt": round(h, 2),
|
|
"width_mm": round(w * POINT_TO_MM, 1),
|
|
"height_mm": round(h * POINT_TO_MM, 1),
|
|
"orientation": "landscape" if w > h else "portrait",
|
|
"looks_a4": _looks_a4(w, h),
|
|
})
|
|
return boxes
|
|
|
|
|
|
def _looks_a4(w: float, h: float, tolerance_pt: float = 20.0) -> bool:
|
|
pairs = [(w, h), (h, w)]
|
|
return any(abs(a - A4_PORTRAIT_PT[0]) <= tolerance_pt and abs(b - A4_PORTRAIT_PT[1]) <= tolerance_pt for a, b in pairs)
|
|
|
|
|
|
def _pdf_info(data: bytes, content_type: str) -> dict[str, Any]:
|
|
info = {
|
|
"content_type": content_type,
|
|
"bytes": len(data),
|
|
"sha256_12": hashlib.sha256(data).hexdigest()[:12] if data else "",
|
|
"is_pdf_header": data[:5] == b"%PDF-",
|
|
"header_preview": data[:80].decode("latin-1", errors="replace"),
|
|
"page_count_hint": None,
|
|
"boxes": [],
|
|
"format_warning": "",
|
|
}
|
|
if not data:
|
|
info["format_warning"] = "empty_response"
|
|
return info
|
|
if not info["is_pdf_header"]:
|
|
info["format_warning"] = "not_pdf_header"
|
|
return info
|
|
latin = data.decode("latin-1", errors="ignore")
|
|
# Count '/Type /Page' but not '/Type /Pages'.
|
|
page_count = len(re.findall(r"/Type\s*/Page(?!s)\b", latin))
|
|
info["page_count_hint"] = page_count or None
|
|
boxes = _extract_pdf_boxes(data)
|
|
info["boxes"] = boxes
|
|
if boxes and not any(bool(b.get("looks_a4")) for b in boxes if b.get("kind", "").lower() == "mediabox"):
|
|
info["format_warning"] = "non_a4_or_unexpected_mediabox"
|
|
elif len(data) < 15_000:
|
|
info["format_warning"] = "very_small_pdf"
|
|
elif len(data) > 2_500_000:
|
|
info["format_warning"] = "very_large_pdf"
|
|
return info
|
|
|
|
|
|
def _query_candidates(args: argparse.Namespace) -> list[dict[str, Any]]:
|
|
where = ["inv.document_kind = 'invoice'", "inv.system = 'jasmin'"]
|
|
params: dict[str, Any] = {"limit": args.limit}
|
|
if args.opportunity_id:
|
|
where.append("inv.opportunity_id = CAST(:opportunity_id AS UUID)")
|
|
params["opportunity_id"] = args.opportunity_id
|
|
if args.document:
|
|
where.append("(inv.document_number = :document OR inv.external_id = :document)")
|
|
params["document"] = args.document
|
|
if args.days and not args.document and not args.opportunity_id:
|
|
where.append("inv.created_at >= now() - (:days * INTERVAL '1 day')")
|
|
params["days"] = int(args.days)
|
|
if not args.include_all_invoices:
|
|
where.append("inv.parent_document_id IS NOT NULL")
|
|
where_sql = " AND ".join(where)
|
|
with engine.begin() as conn:
|
|
rows = conn.execute(text(f"""
|
|
SELECT
|
|
inv.id::text AS invoice_doc_id,
|
|
inv.opportunity_id::text AS opportunity_id,
|
|
inv.external_id AS invoice_external_id,
|
|
inv.document_number AS invoice_number,
|
|
inv.document_type AS invoice_type,
|
|
inv.serie AS invoice_serie,
|
|
inv.series_number AS invoice_series_number,
|
|
inv.status AS invoice_status,
|
|
inv.total_amount AS invoice_total,
|
|
inv.currency AS invoice_currency,
|
|
inv.payload AS invoice_payload,
|
|
inv.created_at AS invoice_created_at,
|
|
q.id::text AS quotation_doc_id,
|
|
q.external_id AS quotation_external_id,
|
|
q.document_number AS quotation_number,
|
|
q.document_type AS quotation_type,
|
|
q.serie AS quotation_serie,
|
|
q.total_amount AS quotation_total,
|
|
q.payload AS quotation_payload,
|
|
o.title AS opportunity_title,
|
|
o.customer_name AS opportunity_customer_name,
|
|
c.name AS fiscal_customer_name,
|
|
c.tax_id AS fiscal_customer_tax_id
|
|
FROM commercial_documents inv
|
|
LEFT JOIN commercial_documents q ON q.id = inv.parent_document_id
|
|
LEFT JOIN opportunities o ON o.id = inv.opportunity_id
|
|
LEFT JOIN customers c ON c.id = inv.customer_id
|
|
WHERE {where_sql}
|
|
ORDER BY inv.created_at DESC
|
|
LIMIT :limit
|
|
"""), params).mappings().all()
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
def _mark_local_pdf_format(result: dict[str, Any]) -> None:
|
|
document_id = _s(result.get("invoice_doc_id"))
|
|
if not document_id:
|
|
return
|
|
info = result.get("invoice_pdf") if isinstance(result.get("invoice_pdf"), dict) else {}
|
|
warning = _s((info or {}).get("format_warning"))
|
|
boxes = (info or {}).get("boxes") or []
|
|
first_box = ""
|
|
if boxes:
|
|
b = boxes[0]
|
|
first_box = f"{b.get('kind')} {b.get('width_mm')}x{b.get('height_mm')}mm a4={b.get('looks_a4')}"
|
|
blocked = warning in {"not_pdf_header", "empty_response", "non_a4_or_unexpected_mediabox"}
|
|
update_commercial_document_details(document_id, {"payload": {
|
|
"clientflow_pdf_format": info or {},
|
|
"clientflow_pdf_format_warning": warning,
|
|
"clientflow_pdf_first_box": first_box,
|
|
"clientflow_pdf_send_blocked": blocked,
|
|
"clientflow_pdf_format_checked_by": "audit_jasmin_converted_invoice_format",
|
|
}})
|
|
|
|
|
|
async def _audit_one(row: dict[str, Any], *, client: JasminClient, save_dir: Path | None, compare_quote_pdf: bool) -> dict[str, Any]:
|
|
invoice_external_id = _s(row.get("invoice_external_id"))
|
|
quotation_external_id = _s(row.get("quotation_external_id"))
|
|
result: dict[str, Any] = {
|
|
**{k: _json_default(v) for k, v in row.items() if not k.endswith("_payload")},
|
|
"invoice_local_payload_keys": sorted(_payload(row.get("invoice_payload")).keys()),
|
|
"quotation_local_payload_keys": sorted(_payload(row.get("quotation_payload")).keys()),
|
|
"remote_invoice": {},
|
|
"remote_quotation": {},
|
|
"invoice_pdf": {},
|
|
"quotation_pdf": {},
|
|
"finding": "OK",
|
|
"finding_detail": "",
|
|
}
|
|
try:
|
|
invoice_raw = await client.get_invoice(invoice_external_id)
|
|
result["remote_invoice"] = _selected_jasmin_fields(invoice_raw)
|
|
except Exception as exc:
|
|
result["finding"] = "REMOTE_INVOICE_ERROR"
|
|
result["finding_detail"] = str(exc)
|
|
return result
|
|
|
|
if quotation_external_id:
|
|
try:
|
|
quotation_raw = await client.get_quotation(quotation_external_id)
|
|
result["remote_quotation"] = _selected_jasmin_fields(quotation_raw)
|
|
except Exception as exc:
|
|
result["remote_quotation_error"] = str(exc)
|
|
|
|
try:
|
|
pdf, content_type = await client.print_invoice_pdf(invoice_external_id)
|
|
result["invoice_pdf"] = _pdf_info(pdf, content_type)
|
|
if save_dir:
|
|
save_dir.mkdir(parents=True, exist_ok=True)
|
|
name = re.sub(r"[^A-Za-z0-9_.-]+", "_", _s(row.get("invoice_number")) or invoice_external_id)
|
|
(save_dir / f"invoice_{name}.pdf").write_bytes(pdf)
|
|
except Exception as exc:
|
|
result["finding"] = "INVOICE_PDF_ERROR"
|
|
result["finding_detail"] = str(exc)
|
|
return result
|
|
|
|
if compare_quote_pdf and quotation_external_id:
|
|
try:
|
|
pdf, content_type = await client.print_quotation_pdf(quotation_external_id)
|
|
result["quotation_pdf"] = _pdf_info(pdf, content_type)
|
|
if save_dir:
|
|
save_dir.mkdir(parents=True, exist_ok=True)
|
|
name = re.sub(r"[^A-Za-z0-9_.-]+", "_", _s(row.get("quotation_number")) or quotation_external_id)
|
|
(save_dir / f"quotation_{name}.pdf").write_bytes(pdf)
|
|
except Exception as exc:
|
|
result["quotation_pdf_error"] = str(exc)
|
|
|
|
inv_pdf = result.get("invoice_pdf") or {}
|
|
warning = _s(inv_pdf.get("format_warning"))
|
|
inv_remote = result.get("remote_invoice") or {}
|
|
if warning:
|
|
result["finding"] = warning.upper()
|
|
result["finding_detail"] = f"PDF warning: {warning}"
|
|
elif not inv_remote.get("documentType") or not inv_remote.get("serie"):
|
|
result["finding"] = "MISSING_REMOTE_TYPE_OR_SERIE"
|
|
result["finding_detail"] = "GET invoice did not expose documentType/serie; check Jasmin payload."
|
|
else:
|
|
# If the remote response exposes print/report/layout-like keys, surface them.
|
|
hints = inv_remote.get("schema_hint_keys") or []
|
|
if hints:
|
|
result["finding_detail"] = "Remote invoice exposes layout/report-related keys: " + ", ".join(hints[:10])
|
|
return result
|
|
|
|
|
|
def _write_outputs(results: list[dict[str, Any]], prefix: str) -> dict[str, str]:
|
|
json_path = f"{prefix}.json"
|
|
md_path = f"{prefix}.md"
|
|
csv_path = f"{prefix}.csv"
|
|
Path(json_path).write_text(json.dumps(results, ensure_ascii=False, indent=2, default=_json_default), encoding="utf-8")
|
|
flat_rows = []
|
|
for r in results:
|
|
inv_pdf = r.get("invoice_pdf") or {}
|
|
boxes = inv_pdf.get("boxes") or []
|
|
first_box = boxes[0] if boxes else {}
|
|
flat_rows.append({
|
|
"finding": r.get("finding"),
|
|
"finding_detail": r.get("finding_detail"),
|
|
"opportunity_id": r.get("opportunity_id"),
|
|
"opportunity_title": r.get("opportunity_title"),
|
|
"fiscal_customer_name": r.get("fiscal_customer_name"),
|
|
"invoice_number": r.get("invoice_number"),
|
|
"invoice_external_id": r.get("invoice_external_id"),
|
|
"invoice_type": r.get("invoice_type"),
|
|
"invoice_serie": r.get("invoice_serie"),
|
|
"remote_invoice_type": _dig(r, "remote_invoice", "documentType"),
|
|
"remote_invoice_serie": _dig(r, "remote_invoice", "serie"),
|
|
"invoice_pdf_content_type": inv_pdf.get("content_type"),
|
|
"invoice_pdf_bytes": inv_pdf.get("bytes"),
|
|
"invoice_pdf_is_pdf_header": inv_pdf.get("is_pdf_header"),
|
|
"invoice_pdf_page_count_hint": inv_pdf.get("page_count_hint"),
|
|
"first_box_kind": first_box.get("kind"),
|
|
"first_box_width_mm": first_box.get("width_mm"),
|
|
"first_box_height_mm": first_box.get("height_mm"),
|
|
"first_box_looks_a4": first_box.get("looks_a4"),
|
|
"quotation_number": r.get("quotation_number"),
|
|
})
|
|
fieldnames = list(flat_rows[0].keys()) if flat_rows else ["finding"]
|
|
with open(csv_path, "w", encoding="utf-8", newline="") as fh:
|
|
writer = csv.DictWriter(fh, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows(flat_rows)
|
|
lines = ["# Auditoria formato PDF faturas Jasmin convertidas", ""]
|
|
for r in results:
|
|
inv_pdf = r.get("invoice_pdf") or {}
|
|
boxes = inv_pdf.get("boxes") or []
|
|
box_text = "; ".join(f"{b.get('kind')} {b.get('width_mm')}x{b.get('height_mm')}mm a4={b.get('looks_a4')}" for b in boxes[:3]) or "—"
|
|
lines.extend([
|
|
f"## {r.get('invoice_number') or r.get('invoice_external_id')}",
|
|
f"- Finding: **{r.get('finding')}** {r.get('finding_detail') or ''}",
|
|
f"- Oportunidade: `{r.get('opportunity_id')}` · {r.get('opportunity_title')}",
|
|
f"- Cliente fiscal: {r.get('fiscal_customer_name') or '—'} · NIF {r.get('fiscal_customer_tax_id') or '—'}",
|
|
f"- Invoice local: type={r.get('invoice_type') or '—'} serie={r.get('invoice_serie') or '—'} total={r.get('invoice_total') or '—'}",
|
|
f"- Invoice remoto: type={_dig(r, 'remote_invoice', 'documentType') or '—'} serie={_dig(r, 'remote_invoice', 'serie') or '—'} number={_dig(r, 'remote_invoice', 'documentNumber') or _dig(r, 'remote_invoice', 'naturalKey') or '—'}",
|
|
f"- PDF: content_type={inv_pdf.get('content_type') or '—'} bytes={inv_pdf.get('bytes') or '—'} pages={inv_pdf.get('page_count_hint') or '—'} boxes={box_text}",
|
|
"",
|
|
])
|
|
Path(md_path).write_text("\n".join(lines), encoding="utf-8")
|
|
return {"json": json_path, "csv": csv_path, "markdown": md_path}
|
|
|
|
|
|
async def _main_async(args: argparse.Namespace) -> int:
|
|
candidates = _query_candidates(args)
|
|
print(f"CONVERTED_INVOICES_TO_SCAN={len(candidates)}")
|
|
client = JasminClient(timeout=float(args.timeout))
|
|
save_dir = Path(args.save_pdf_dir) if args.save_pdf_dir else None
|
|
results: list[dict[str, Any]] = []
|
|
for i, row in enumerate(candidates, start=1):
|
|
print(f"scanning={i}/{len(candidates)} invoice={row.get('invoice_number') or row.get('invoice_external_id')} opp={row.get('opportunity_id')} title={row.get('opportunity_title')}")
|
|
result = await _audit_one(row, client=client, save_dir=save_dir, compare_quote_pdf=not args.no_quote_pdf)
|
|
results.append(result)
|
|
if getattr(args, "mark_local", False):
|
|
try:
|
|
_mark_local_pdf_format(result)
|
|
except Exception as exc:
|
|
print(f" mark_local_error={exc}")
|
|
inv_pdf = result.get("invoice_pdf") or {}
|
|
boxes = inv_pdf.get("boxes") or []
|
|
box = boxes[0] if boxes else {}
|
|
print(
|
|
" finding={finding} content_type={ct} bytes={bytes} box={w}x{h}mm a4={a4}".format(
|
|
finding=result.get("finding"),
|
|
ct=inv_pdf.get("content_type"),
|
|
bytes=inv_pdf.get("bytes"),
|
|
w=box.get("width_mm"),
|
|
h=box.get("height_mm"),
|
|
a4=box.get("looks_a4"),
|
|
)
|
|
)
|
|
prefix = args.output_prefix or "/tmp/clientflow_jasmin_converted_invoice_format_audit"
|
|
paths = _write_outputs(results, prefix)
|
|
counts: dict[str, int] = {}
|
|
for r in results:
|
|
counts[_s(r.get("finding") or "UNKNOWN")] = counts.get(_s(r.get("finding") or "UNKNOWN"), 0) + 1
|
|
print("SUMMARY")
|
|
print(f"Findings: {len([r for r in results if r.get('finding') != 'OK'])}")
|
|
for key in sorted(counts):
|
|
print(f"{key}: {counts[key]}")
|
|
print(f"CSV: {paths['csv']}")
|
|
print(f"Markdown: {paths['markdown']}")
|
|
print(f"JSON: {paths['json']}")
|
|
if save_dir:
|
|
print(f"PDF_DIR: {save_dir}")
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Audit format/size of Jasmin PDFs generated after ORC→FA conversion.")
|
|
parser.add_argument("--days", type=int, default=14, help="Recent invoice window when no document/opportunity filter is supplied.")
|
|
parser.add_argument("--limit", type=int, default=30)
|
|
parser.add_argument("--document", help="Invoice document_number or external_id, e.g. FA.FA2026.135")
|
|
parser.add_argument("--opportunity-id", help="Limit to one opportunity UUID")
|
|
parser.add_argument("--include-all-invoices", action="store_true", help="Also scan invoices not linked to a parent quotation.")
|
|
parser.add_argument("--no-quote-pdf", action="store_true", help="Do not fetch parent quotation PDF for comparison.")
|
|
parser.add_argument("--save-pdf-dir", help="Optional directory to save fetched PDFs for manual inspection.")
|
|
parser.add_argument("--mark-local", action="store_true", help="Persist local PDF format flags in commercial_documents.payload; does not modify Jasmin.")
|
|
parser.add_argument("--output-prefix", default="/tmp/clientflow_jasmin_converted_invoice_format_audit")
|
|
parser.add_argument("--timeout", type=float, default=45.0)
|
|
args = parser.parse_args()
|
|
return asyncio.run(_main_async(args))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|