Coverage for jbank/management/commands/parse_xt.py: 47%
109 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-27 13:36 +0700
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-27 13:36 +0700
1import logging
2import os
3from pprint import pprint
5from django.core.exceptions import ValidationError
6from django.core.management.base import CommandParser
7from django.db import transaction
8from jbank.camt import (
9 camt053_get_iban,
10 camt053_create_statement,
11 camt053_parse_statement_from_file,
12 CAMT053_STATEMENT_SUFFIXES,
13 camt053_get_unified_str,
14)
15from jbank.helpers import get_or_create_bank_account, save_or_store_media
16from jbank.files import list_dir_files
17from jbank.models import Statement, StatementFile, StatementRecord, StatementRecordDetail
18from jbank.parsers import parse_filename_suffix
19from jutil.command import SafeCommand
21logger = logging.getLogger(__name__)
24class Command(SafeCommand):
25 help = "Parses XML bank account statement (camt.053.001.02) files"
27 def add_arguments(self, parser: CommandParser):
28 parser.add_argument("path", type=str)
29 parser.add_argument("--verbose", action="store_true")
30 parser.add_argument("--test", action="store_true")
31 parser.add_argument("--delete-old", action="store_true")
32 parser.add_argument("--auto-create-accounts", action="store_true")
33 parser.add_argument("--suffix", type=str)
34 parser.add_argument("--resolve-original-filenames", action="store_true")
35 parser.add_argument("--tag", type=str, default="")
36 parser.add_argument("--parse-creditor-account-data", action="store_true", help="For data migration")
38 def parse_creditor_account_data(self): # pylint: disable=too-many-locals,too-many-branches
39 for sf in StatementFile.objects.all(): # pylint: disable=too-many-nested-blocks
40 assert isinstance(sf, StatementFile)
41 full_path = sf.full_path
42 if os.path.isfile(full_path) and parse_filename_suffix(full_path).upper() in CAMT053_STATEMENT_SUFFIXES:
43 logger.info("Parsing creditor account data of %s", full_path)
44 statement_data = camt053_parse_statement_from_file(full_path)
45 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {})
46 d_ntry = d_stmt.get("Ntry", [])
47 recs = list(StatementRecord.objects.all().filter(statement__file=sf).order_by("id"))
48 if len(recs) != len(d_ntry):
49 raise ValidationError(f"Statement record counts do not match in id={sf.id} ({sf})")
50 for ix, ntry in enumerate(d_ntry):
51 rec = recs[ix]
52 assert isinstance(rec, StatementRecord)
53 for dtl_batch in ntry.get("NtryDtls", []):
54 rec_detail_list = list(StatementRecordDetail.objects.all().filter(record=rec))
55 if len(rec_detail_list) != len(dtl_batch.get("TxDtls", [])):
56 raise ValidationError(f"Statement record detail counts do not match in id={sf.id} ({sf})")
57 for dtl_ix, dtl in enumerate(dtl_batch.get("TxDtls", [])):
58 d = rec_detail_list[dtl_ix]
59 assert isinstance(d, StatementRecordDetail)
60 d_parties = dtl.get("RltdPties", {})
61 d_dbt = d_parties.get("Dbtr", {})
62 d.debtor_name = d_dbt.get("Nm", "")
63 d_udbt = d_parties.get("UltmtDbtr", {})
64 d.ultimate_debtor_name = d_udbt.get("Nm", "")
65 d_cdtr = d_parties.get("Cdtr", {})
66 d.creditor_name = d_cdtr.get("Nm", "")
67 d_cdtr_acct = d_parties.get("CdtrAcct", {})
68 d_cdtr_acct_id = d_cdtr_acct.get("Id", {})
69 d.creditor_account = d_cdtr_acct_id.get("IBAN", "")
70 if d.creditor_account:
71 d.creditor_account_scheme = "IBAN"
72 else:
73 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {}
74 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "")
75 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or ""
76 logger.info("%s creditor_account %s (%s)", rec, d.creditor_account, d.creditor_account_scheme)
77 d.save()
79 if not rec.recipient_account_number:
80 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account")
81 if rec.recipient_account_number:
82 rec.save(update_fields=["recipient_account_number"])
83 logger.info("%s recipient_account_number %s", rec, rec.recipient_account_number)
85 def do(self, *args, **options): # pylint: disable=too-many-branches
86 if options["parse_creditor_account_data"]:
87 self.parse_creditor_account_data()
88 return
90 files = list_dir_files(options["path"], options["suffix"])
91 for filename in files:
92 plain_filename = os.path.basename(filename)
94 if parse_filename_suffix(plain_filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
95 print("Ignoring non-CAMT53 file {}".format(filename))
96 continue
98 if options["resolve_original_filenames"]:
99 found = StatementFile.objects.filter(statement__name=plain_filename).first()
100 if found and not found.original_filename:
101 assert isinstance(found, StatementFile)
102 found.original_filename = filename
103 found.save(update_fields=["original_filename"])
104 logger.info("Original XML statement filename of %s resolved to %s", found, filename)
106 if options["test"]:
107 statement = camt053_parse_statement_from_file(filename)
108 pprint(statement)
109 continue
111 if options["delete_old"]:
112 Statement.objects.filter(name=plain_filename).delete()
114 if not Statement.objects.filter(name=plain_filename).first():
115 print("Importing statement file {}".format(plain_filename))
117 statement = camt053_parse_statement_from_file(filename)
118 if options["verbose"]:
119 pprint(statement)
121 with transaction.atomic():
122 file = StatementFile(original_filename=filename, tag=options["tag"])
123 file.save()
124 save_or_store_media(file.file, filename)
125 file.save()
127 for data in [statement]:
128 if options["auto_create_accounts"]:
129 account_number = camt053_get_iban(data)
130 if account_number:
131 get_or_create_bank_account(account_number)
133 camt053_create_statement(data, name=plain_filename, file=file) # pytype: disable=not-callable
134 else:
135 print("Skipping statement file {}".format(filename))