feat: CSV-Import ueber bestehende Pipeline mit dreiwertiger Saldo-Pruefung
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -5,7 +7,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models.tables import Account, Statement, Transaction
|
||||
from app.parsers.base import ParserError
|
||||
from app.parsers.base import ParsedStatement, ParserError
|
||||
from app.parsers.csv_formats import parse_csv
|
||||
from app.parsers.registry import parse_pdf
|
||||
from app.parsers.validate import balance_difference, dedup_hash
|
||||
from app.services.categorize import apply_rules
|
||||
@@ -14,6 +17,17 @@ from app.services.categorize import apply_rules
|
||||
def _find_or_create_account(session: Session, bank: str, iban: str | None, filename: str) -> Account:
|
||||
if not iban:
|
||||
iban = f"UNBEKANNT-{filename}"
|
||||
if bank == "hvb_csv":
|
||||
# HVB-CSV traegt nur die Kontonummer (keine IBAN). Konto-Zuordnung
|
||||
# ueber "endet auf Kontonummer" gegen alle bestehenden IBANs -
|
||||
# dokumentierte Einschraenkung (siehe Formatreferenz Task 2/3).
|
||||
for existing in session.execute(select(Account)).scalars():
|
||||
if existing.iban.endswith(iban):
|
||||
return existing
|
||||
acc = Account(bank=bank, iban=iban, name=iban, type="giro")
|
||||
session.add(acc)
|
||||
session.flush()
|
||||
return acc
|
||||
acc = session.execute(select(Account).where(Account.iban == iban)).scalar()
|
||||
if acc is not None:
|
||||
return acc
|
||||
@@ -23,6 +37,20 @@ def _find_or_create_account(session: Session, bank: str, iban: str | None, filen
|
||||
return acc
|
||||
|
||||
|
||||
def _apply_anchor_autofill(account: Account, anchor: tuple[date, Decimal] | None) -> None:
|
||||
"""CSV-Anker-Regel (Ausbaustufe 3 Task 3): der Datei-Kontostand wird nur
|
||||
uebernommen, wenn er neuer ist als ein evtl. vorhandener Anker (oder noch
|
||||
keiner existiert). Ein neuerer, manuell per PATCH gesetzter Anker bleibt
|
||||
unangetastet - so entkommt kein aelterer CSV-Export einem bewusst
|
||||
gesetzten aktuellen Stand."""
|
||||
if anchor is None:
|
||||
return
|
||||
anchor_date, anchor_balance = anchor
|
||||
if account.anchor_date is None or anchor_date >= account.anchor_date:
|
||||
account.anchor_date = anchor_date
|
||||
account.anchor_balance = anchor_balance
|
||||
|
||||
|
||||
def _error_statement(session: Session, filename: str, message: str) -> Statement:
|
||||
stmt = Statement(filename=filename, bank="unbekannt", account_id=None,
|
||||
status="error", error_message=message)
|
||||
@@ -32,23 +60,18 @@ def _error_statement(session: Session, filename: str, message: str) -> Statement
|
||||
return stmt
|
||||
|
||||
|
||||
def process_pdf(session: Session, path: Path) -> Statement:
|
||||
def _finish_statement(session: Session, path: Path, parsed: ParsedStatement,
|
||||
check_balance: bool,
|
||||
anchor: tuple[date, Decimal] | None) -> Statement:
|
||||
"""Gemeinsamer Abschluss fuer PDF- und CSV-Importe, sobald ein
|
||||
ParsedStatement vorliegt: Konto zuordnen, Statement+Transaktionen als
|
||||
Entwurf anlegen, optional den Saldo pruefen, Datei aus dem Posteingang
|
||||
verschieben. `check_balance=False` (CSV-Formate ohne Saldodaten, z.B.
|
||||
HVB/DKB) laesst den Entwurf OHNE Fehler stehen statt einer
|
||||
Saldo-Differenz-Pruefung, die auf None-Feldern crashen wuerde."""
|
||||
settings = get_settings()
|
||||
filename = path.name
|
||||
|
||||
try:
|
||||
parsed = parse_pdf(path)
|
||||
except ParserError as exc:
|
||||
return _error_statement(session, filename, str(exc))
|
||||
except Exception as exc:
|
||||
# Fremdbibliotheken (pdfplumber/pdfminer) werfen bei strukturell
|
||||
# kaputten PDFs eigene Exception-Typen statt ParserError (z.B.
|
||||
# PdfminerException "No /Root object!"). Ohne diesen Fang wuerde ein
|
||||
# kaputtes Upload zu einem unbehandelten 500 fuehren statt zu einem
|
||||
# sauberen Fehler-Import wie bei "Bank nicht erkannt". Die Datei
|
||||
# bleibt dabei unangetastet im Posteingang.
|
||||
return _error_statement(session, filename, f"PDF nicht lesbar: {exc}")
|
||||
|
||||
account = _find_or_create_account(session, parsed.bank, parsed.iban, filename)
|
||||
|
||||
stmt = Statement(
|
||||
@@ -64,13 +87,14 @@ def process_pdf(session: Session, path: Path) -> Statement:
|
||||
session.add(stmt)
|
||||
session.flush()
|
||||
|
||||
diff = balance_difference(parsed)
|
||||
if diff != 0:
|
||||
stmt.status = "error"
|
||||
stmt.error_message = f"Saldo-Differenz {diff} EUR"
|
||||
session.commit()
|
||||
session.refresh(stmt)
|
||||
return stmt
|
||||
if check_balance:
|
||||
diff = balance_difference(parsed)
|
||||
if diff != 0:
|
||||
stmt.status = "error"
|
||||
stmt.error_message = f"Saldo-Differenz {diff} EUR"
|
||||
session.commit()
|
||||
session.refresh(stmt)
|
||||
return stmt
|
||||
|
||||
drafts = []
|
||||
for t in parsed.transactions:
|
||||
@@ -100,10 +124,11 @@ def process_pdf(session: Session, path: Path) -> Statement:
|
||||
apply_rules(session, drafts)
|
||||
|
||||
stmt.status = "draft"
|
||||
_apply_anchor_autofill(account, anchor)
|
||||
|
||||
# Invariant: a committed draft implies the PDF left the inbox. Move the
|
||||
# file before committing so a failed move can never leave a committed
|
||||
# draft with the PDF still sitting in the inbox.
|
||||
# Invariant: a committed draft implies the source file left the inbox.
|
||||
# Move the file before committing so a failed move can never leave a
|
||||
# committed draft with the file still sitting in the inbox.
|
||||
uploads_dir = settings.uploads_dir
|
||||
uploads_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
@@ -116,3 +141,46 @@ def process_pdf(session: Session, path: Path) -> Statement:
|
||||
session.refresh(stmt)
|
||||
|
||||
return stmt
|
||||
|
||||
|
||||
def process_pdf(session: Session, path: Path) -> Statement:
|
||||
filename = path.name
|
||||
try:
|
||||
parsed = parse_pdf(path)
|
||||
except ParserError as exc:
|
||||
return _error_statement(session, filename, str(exc))
|
||||
except Exception as exc:
|
||||
# Fremdbibliotheken (pdfplumber/pdfminer) werfen bei strukturell
|
||||
# kaputten PDFs eigene Exception-Typen statt ParserError (z.B.
|
||||
# PdfminerException "No /Root object!"). Ohne diesen Fang wuerde ein
|
||||
# kaputtes Upload zu einem unbehandelten 500 fuehren statt zu einem
|
||||
# sauberen Fehler-Import wie bei "Bank nicht erkannt". Die Datei
|
||||
# bleibt dabei unangetastet im Posteingang.
|
||||
return _error_statement(session, filename, f"PDF nicht lesbar: {exc}")
|
||||
|
||||
return _finish_statement(session, path, parsed, check_balance=True, anchor=None)
|
||||
|
||||
|
||||
def process_csv(session: Session, path: Path) -> Statement:
|
||||
filename = path.name
|
||||
try:
|
||||
parsed_csv = parse_csv(path)
|
||||
except ParserError as exc:
|
||||
return _error_statement(session, filename, str(exc))
|
||||
except Exception as exc:
|
||||
# Analog zu process_pdf: eine strukturell kaputte/unlesbare CSV darf
|
||||
# nicht als unbehandelter 500 durchschlagen.
|
||||
return _error_statement(session, filename, f"CSV nicht lesbar: {exc}")
|
||||
|
||||
return _finish_statement(session, path, parsed_csv.statement,
|
||||
check_balance=parsed_csv.balance_checkable,
|
||||
anchor=parsed_csv.anchor)
|
||||
|
||||
|
||||
def process_file(session: Session, path: Path) -> Statement:
|
||||
"""Dateiendungs-Weiche fuer die Import-Pipeline (Ausbaustufe 3 Task 3):
|
||||
.csv laeuft ueber den CSV-Formatparser, alles andere weiterhin ueber den
|
||||
PDF-Parser."""
|
||||
if path.suffix.lower() == ".csv":
|
||||
return process_csv(session, path)
|
||||
return process_pdf(session, path)
|
||||
|
||||
Reference in New Issue
Block a user