Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# pylint: disable=too-many-branches 

2import logging 

3import os 

4from pprint import pprint 

5 

6from django.core.exceptions import ValidationError 

7from django.core.management.base import CommandParser 

8from django.db import transaction 

9from jbank.camt import ( 

10 camt053_get_iban, 

11 camt053_create_statement, 

12 camt053_parse_statement_from_file, 

13 CAMT053_STATEMENT_SUFFIXES, 

14 camt053_get_unified_str, 

15) 

16from jbank.helpers import get_or_create_bank_account, save_or_store_media 

17from jbank.files import list_dir_files 

18from jbank.models import Statement, StatementFile, StatementRecord, StatementRecordDetail 

19from jbank.parsers import parse_filename_suffix 

20from jutil.command import SafeCommand 

21 

22logger = logging.getLogger(__name__) 

23 

24 

25class Command(SafeCommand): 

26 help = "Parses XML bank account statement (camt.053.001.02) files" 

27 

28 def add_arguments(self, parser: CommandParser): 

29 parser.add_argument("path", type=str) 

30 parser.add_argument("--verbose", action="store_true") 

31 parser.add_argument("--test", action="store_true") 

32 parser.add_argument("--delete-old", action="store_true") 

33 parser.add_argument("--auto-create-accounts", action="store_true") 

34 parser.add_argument("--suffix", type=str) 

35 parser.add_argument("--resolve-original-filenames", action="store_true") 

36 parser.add_argument("--tag", type=str, default="") 

37 parser.add_argument("--parse-creditor-account-data", action="store_true", help="For data migration") 

38 

39 def parse_creditor_account_data(self): # pylint: disable=too-many-locals 

40 for sf in StatementFile.objects.all(): # pylint: disable=too-many-nested-blocks 

41 assert isinstance(sf, StatementFile) 

42 full_path = sf.full_path 

43 if os.path.isfile(full_path) and parse_filename_suffix(full_path).upper() in CAMT053_STATEMENT_SUFFIXES: 

44 logger.info("Parsing creditor account data of %s", full_path) 

45 statement_data = camt053_parse_statement_from_file(full_path) 

46 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {}) 

47 d_ntry = d_stmt.get("Ntry", []) 

48 recs = list(StatementRecord.objects.all().filter(statement__file=sf).order_by("id")) 

49 if len(recs) != len(d_ntry): 

50 raise ValidationError(f"Statement record counts do not match in id={sf.id} ({sf})") 

51 for ix, ntry in enumerate(d_ntry): 

52 rec = recs[ix] 

53 assert isinstance(rec, StatementRecord) 

54 for dtl_batch in ntry.get("NtryDtls", []): 

55 rec_detail_list = list(StatementRecordDetail.objects.all().filter(record=rec)) 

56 if len(rec_detail_list) != len(dtl_batch.get("TxDtls", [])): 

57 raise ValidationError(f"Statement record detail counts do not match in id={sf.id} ({sf})") 

58 for dtl_ix, dtl in enumerate(dtl_batch.get("TxDtls", [])): 

59 d = rec_detail_list[dtl_ix] 

60 assert isinstance(d, StatementRecordDetail) 

61 d_parties = dtl.get("RltdPties", {}) 

62 d_dbt = d_parties.get("Dbtr", {}) 

63 d.debtor_name = d_dbt.get("Nm", "") 

64 d_udbt = d_parties.get("UltmtDbtr", {}) 

65 d.ultimate_debtor_name = d_udbt.get("Nm", "") 

66 d_cdtr = d_parties.get("Cdtr", {}) 

67 d.creditor_name = d_cdtr.get("Nm", "") 

68 d_cdtr_acct = d_parties.get("CdtrAcct", {}) 

69 d_cdtr_acct_id = d_cdtr_acct.get("Id", {}) 

70 d.creditor_account = d_cdtr_acct_id.get("IBAN", "") 

71 if d.creditor_account: 

72 d.creditor_account_scheme = "IBAN" 

73 else: 

74 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {} 

75 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "") 

76 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or "" 

77 logger.info( 

78 "%s creditor_account %s (%s)", rec, d.creditor_account, d.creditor_account_scheme 

79 ) 

80 d.save() 

81 

82 if not rec.recipient_account_number: 

83 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account") 

84 if rec.recipient_account_number: 

85 rec.save(update_fields=["recipient_account_number"]) 

86 logger.info("%s recipient_account_number %s", rec, rec.recipient_account_number) 

87 

88 def do(self, *args, **options): 

89 if options["parse_creditor_account_data"]: 

90 self.parse_creditor_account_data() 

91 return 

92 

93 files = list_dir_files(options["path"], options["suffix"]) 

94 for filename in files: # pylint: disable=too-many-nested-blocks 

95 plain_filename = os.path.basename(filename) 

96 

97 if parse_filename_suffix(plain_filename).upper() not in CAMT053_STATEMENT_SUFFIXES: 

98 print("Ignoring non-CAMT53 file {}".format(filename)) 

99 continue 

100 

101 if options["resolve_original_filenames"]: 

102 found = StatementFile.objects.filter(statement__name=plain_filename).first() 

103 if found and not found.original_filename: 

104 assert isinstance(found, StatementFile) 

105 found.original_filename = filename 

106 found.save(update_fields=["original_filename"]) 

107 logger.info("Original XML statement filename of %s resolved to %s", found, filename) 

108 

109 if options["test"]: 

110 statement = camt053_parse_statement_from_file(filename) 

111 pprint(statement) 

112 continue 

113 

114 if options["delete_old"]: 

115 Statement.objects.filter(name=plain_filename).delete() 

116 

117 if not Statement.objects.filter(name=plain_filename).first(): 

118 print("Importing statement file {}".format(plain_filename)) 

119 

120 statement = camt053_parse_statement_from_file(filename) 

121 if options["verbose"]: 

122 pprint(statement) 

123 

124 with transaction.atomic(): 

125 file = StatementFile(original_filename=filename, tag=options["tag"]) 

126 file.save() 

127 save_or_store_media(file.file, filename) 

128 file.save() 

129 

130 for data in [statement]: 

131 if options["auto_create_accounts"]: 

132 account_number = camt053_get_iban(data) 

133 if account_number: 

134 get_or_create_bank_account(account_number) 

135 

136 camt053_create_statement(data, name=plain_filename, file=file) # pytype: disable=not-callable 

137 else: 

138 print("Skipping statement file {}".format(filename))