Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from copy import copy 

2from os.path import basename 

3from typing import Dict, Any, List 

4from django.core.exceptions import ValidationError 

5from django.utils.translation import gettext as _ 

6from pytz import timezone 

7from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields 

8 

9TO_STATEMENT_SUFFIXES = ("TO", "TXT", "TITO") 

10 

11TO_FILE_HEADER_TYPES = ("T00",) 

12 

13TO_FILE_HEADER_DATES = ( 

14 "begin_date", 

15 "end_date", 

16 ("record_date", "record_time"), 

17 "begin_balance_date", 

18) 

19 

20TO_FILE_HEADER_DECIMALS = ( 

21 ("begin_balance", "begin_balance_sign"), 

22 "account_limit", 

23) 

24 

25TO_FILE_HEADER = ( 

26 ("statement_type", "X", "P"), 

27 ("record_type", "XX", "P"), 

28 ("record_length", "9(3)", "P"), 

29 ("version", "X(3)", "P"), 

30 ("account_number", "X(14)", "P"), 

31 ("statement_number", "9(3)", "P"), 

32 ("begin_date", "9(6)", "P"), 

33 ("end_date", "9(6)", "P"), 

34 ("record_date", "9(6)", "P"), 

35 ("record_time", "9(4)", "P"), 

36 ("customer_identifier", "X(17)", "P"), 

37 ("begin_balance_date", "9(6)", "P"), 

38 ("begin_balance_sign", "X", "P"), 

39 ("begin_balance", "9(18)", "P"), 

40 ("record_count", "9(6)", "P"), 

41 ("currency_code", "X(3)", "P"), 

42 ("account_name", "X(30)", "V"), 

43 ("account_limit", "9(18)", "P"), 

44 ("owner_name", "X(35)", "P"), 

45 ("contact_info_1", "X(40)", "P"), 

46 ("contact_info_2", "X(40)", "V"), 

47 ("bank_specific_info_1", "X(30)", "V"), 

48 ("iban_and_bic", "X(30)", "V"), 

49) 

50 

51TO_FILE_RECORD_TYPES = ("T10", "T80") 

52 

53TO_FILE_RECORD_DATES = ( 

54 "record_date", 

55 "value_date", 

56 "paid_date", 

57) 

58 

59TO_FILE_RECORD_DECIMALS = (("amount", "amount_sign"),) 

60 

61TO_FILE_RECORD = ( 

62 ("statement_type", "X", "P"), 

63 ("record_type", "XX", "P"), 

64 ("record_length", "9(3)", "P"), 

65 ("record_number", "9(6)", "P"), 

66 ("archive_identifier", "X(18)", "V"), 

67 ("record_date", "9(6)", "P"), 

68 ("value_date", "9(6)", "V"), 

69 ("paid_date", "9(6)", "V"), 

70 ("entry_type", "X", "P"), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma 

71 ("record_code", "X(3)", "P"), 

72 ("record_description", "X(35)", "P"), 

73 ("amount_sign", "X", "P"), 

74 ("amount", "9(18)", "P"), 

75 ("receipt_code", "X", "P"), 

76 ("delivery_method", "X", "P"), 

77 ("name", "X(35)", "V"), 

78 ("name_source", "X", "V"), 

79 ("recipient_account_number", "X(14)", "V"), 

80 ("recipient_account_number_changed", "X", "V"), 

81 ("remittance_info", "X(20)", "V"), 

82 ("form_number", "X(8)", "V"), 

83 ("level_identifier", "X", "P"), 

84) 

85 

86TO_FILE_RECORD_EXTRA_INFO_TYPES = ("T11", "T81") 

87 

88TO_FILE_RECORD_EXTRA_INFO_HEADER = ( 

89 ("statement_type", "X", "P"), 

90 ("record_type", "XX", "P"), 

91 ("record_length", "9(3)", "P"), 

92 ("extra_info_type", "9(2)", "P"), 

93) 

94 

95TO_FILE_RECORD_EXTRA_INFO_COUNTS = (("entry_count", "9(8)", "P"),) 

96 

97TO_FILE_RECORD_EXTRA_INFO_INVOICE = ( 

98 ("customer_number", "X(10)", "P"), 

99 ("pad01", "X", "P"), 

100 ("invoice_number", "X(15)", "P"), 

101 ("pad02", "X", "P"), 

102 ("invoice_date", "X(6)", "P"), 

103) 

104 

105TO_FILE_RECORD_EXTRA_INFO_CARD = ( 

106 ("card_number", "X(19)", "P"), 

107 ("pad01", "X", "P"), 

108 ("merchant_reference", "X(14)", "P"), 

109) 

110 

111TO_FILE_RECORD_EXTRA_INFO_CORRECTION = (("original_archive_identifier", "X(18)", "P"),) 

112 

113TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = (("amount", "amount_sign"),) 

114 

115TO_FILE_RECORD_EXTRA_INFO_CURRENCY = ( 

116 ("amount_sign", "X", "P"), 

117 ("amount", "9(18)", "P"), 

118 ("pad01", "X", "P"), 

119 ("currency_code", "X(3)", "P"), 

120 ("pad02", "X", "P"), 

121 ("exchange_rate", "9(11)", "P"), 

122 ("rate_reference", "X(6)", "V"), 

123) 

124 

125TO_FILE_RECORD_EXTRA_INFO_REASON = ( 

126 ("reason_code", "9(3)", "P"), 

127 ("pad01", "X", "P"), 

128 ("reason_description", "X(31)", "P"), 

129) 

130 

131TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = (("name_detail", "X(35)", "P"),) 

132 

133TO_FILE_RECORD_EXTRA_INFO_SEPA = ( 

134 ("reference", "X(35)", "V"), 

135 ("iban_account_number", "X(35)", "V"), 

136 ("bic_code", "X(35)", "V"), 

137 ("recipient_name_detail", "X(70)", "V"), 

138 ("payer_name_detail", "X(70)", "V"), 

139 ("identifier", "X(35)", "V"), 

140 ("archive_identifier", "X(35)", "V"), 

141) 

142 

143TO_BALANCE_RECORD_DATES = ("record_date",) 

144 

145TO_BALANCE_RECORD_DECIMALS = ( 

146 ("end_balance", "end_balance_sign"), 

147 ("available_balance", "available_balance_sign"), 

148) 

149 

150TO_BALANCE_RECORD = ( 

151 ("statement_type", "X", "P"), 

152 ("record_type", "XX", "P"), 

153 ("record_length", "9(3)", "P"), 

154 ("record_date", "9(6)", "P"), 

155 ("end_balance_sign", "X", "P"), 

156 ("end_balance", "9(18)", "P"), 

157 ("available_balance_sign", "X", "P"), 

158 ("available_balance", "9(18)", "P"), 

159) 

160 

161TO_CUMULATIVE_RECORD_DATES = ("period_date",) 

162 

163TO_CUMULATIVE_RECORD_DECIMALS = ( 

164 ("deposits_amount", "deposits_sign"), 

165 ("withdrawals_amount", "withdrawals_sign"), 

166) 

167 

168TO_CUMULATIVE_RECORD = ( 

169 ("statement_type", "X", "P"), 

170 ("record_type", "XX", "P"), 

171 ("record_length", "9(3)", "P"), 

172 ("period_identifier", "X", "P"), # 1=day, 2=term, 3=month, 4=year 

173 ("period_date", "9(6)", "P"), 

174 ("deposits_count", "9(8)", "P"), 

175 ("deposits_sign", "X", "P"), 

176 ("deposits_amount", "9(18)", "P"), 

177 ("withdrawals_count", "9(8)", "P"), 

178 ("withdrawals_sign", "X", "P"), 

179 ("withdrawals_amount", "9(18)", "P"), 

180) 

181 

182TO_SPECIAL_RECORD = ( 

183 ("statement_type", "X", "P"), 

184 ("record_type", "XX", "P"), 

185 ("record_length", "9(3)", "P"), 

186 ("bank_group_identifier", "X(3)", "P"), 

187) 

188 

189 

190def parse_tiliote_statements_from_file(filename: str) -> list: 

191 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES: 

192 raise ValidationError( 

193 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format( 

194 filename=filename, suffixes=", ".join(TO_STATEMENT_SUFFIXES), file_type="tiliote" 

195 ) 

196 ) 

197 with open(filename, "rt", encoding="ISO-8859-1") as fp: 

198 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore 

199 

200 

201def parse_tiliote_statements(content: str, filename: str) -> List[dict]: # pylint: disable=too-many-locals 

202 lines = content.split("\n") 

203 nlines = len(lines) 

204 

205 line_number = 1 

206 tz = timezone("Europe/Helsinki") 

207 

208 header = None 

209 records = [] 

210 balance = None 

211 cumulative = None 

212 cumulative_adjustment = None 

213 special_records = [] 

214 statements = [] 

215 

216 while line_number <= nlines: 

217 line = lines[line_number - 1] 

218 if line.strip() == "": 

219 line_number += 1 

220 continue 

221 # print('parsing line {}: {}'.format(line_number, line)) 

222 record_type = line[:3] 

223 

224 if record_type in TO_FILE_HEADER_TYPES: 

225 if header: 

226 statements.append( 

227 combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records) 

228 ) 

229 header, records, balance, cumulative, cumulative_adjustment, special_records = ( 

230 None, 

231 [], 

232 None, 

233 None, 

234 None, 

235 [], 

236 ) 

237 

238 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number) 

239 convert_date_fields(header, TO_FILE_HEADER_DATES, tz) 

240 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS) 

241 iban_and_bic = str(header.get("iban_and_bic", "")).split(" ") 

242 if len(iban_and_bic) == 2: 

243 header["iban"], header["bic"] = iban_and_bic 

244 line_number += 1 

245 

246 elif record_type in TO_FILE_RECORD_TYPES: 

247 record = parse_records(line, TO_FILE_RECORD, line_number=line_number) 

248 convert_date_fields(record, TO_FILE_RECORD_DATES, tz) 

249 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS) 

250 

251 line_number += 1 

252 # check for record extra info 

253 if line_number <= nlines: 

254 line = lines[line_number - 1] 

255 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES: 

256 parse_record_extra_info(record, line, line_number=line_number) 

257 line_number += 1 

258 line = lines[line_number - 1] 

259 

260 records.append(record) 

261 elif record_type == "T40": 

262 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number) 

263 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz) 

264 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS) 

265 line_number += 1 

266 elif record_type == "T50": 

267 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number) 

268 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz) 

269 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS) 

270 line_number += 1 

271 elif record_type == "T51": 

272 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number) 

273 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz) 

274 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS) 

275 line_number += 1 

276 elif record_type in ("T60", "T70"): 

277 special_records.append( 

278 parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False) 

279 ) 

280 line_number += 1 

281 else: 

282 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type)) 

283 

284 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records)) 

285 return statements 

286 

287 

288def combine_statement( # pylint: disable=too-many-arguments 

289 header, records, balance, cumulative, cumulative_adjustment, special_records 

290) -> Dict[str, Any]: 

291 data = { 

292 "header": header, 

293 "records": records, 

294 } 

295 if balance is not None: 

296 data["balance"] = balance 

297 if cumulative is not None: 

298 data["cumulative"] = cumulative 

299 if cumulative_adjustment is not None: 

300 data["cumulative_adjustment"] = cumulative_adjustment 

301 if special_records: 

302 data["special_records"] = special_records 

303 return data 

304 

305 

306def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int): 

307 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES: 

308 raise ValidationError("SVM record extra info validation error on line {}".format(line_number)) 

309 

310 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False) 

311 extra_info_type = header["extra_info_type"] 

312 # print(line) 

313 extra_data = copy(header["extra_data"]) 

314 assert isinstance(extra_data, str) 

315 if extra_info_type == "00": 

316 record["messages"] = parse_record_messages(extra_data) 

317 elif extra_info_type == "01": 

318 record["counts"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8) 

319 elif extra_info_type == "02": 

320 record["invoice"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33) 

321 elif extra_info_type == "03": 

322 record["card"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34) 

323 elif extra_info_type == "04": 

324 record["correction"] = parse_records( 

325 extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18 

326 ) 

327 elif extra_info_type == "05": 

328 record["currency"] = parse_records( 

329 extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41 

330 ) 

331 convert_decimal_fields(record["currency"], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS) 

332 elif extra_info_type == "06": 

333 record["client_messages"] = parse_record_messages(extra_data) 

334 elif extra_info_type == "07": 

335 record["bank_messages"] = parse_record_messages(extra_data) 

336 elif extra_info_type == "08": 

337 record["reason"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35) 

338 elif extra_info_type == "09": 

339 record["name_detail"] = parse_records( 

340 extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35 

341 ) 

342 elif extra_info_type == "11": 

343 record["sepa"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323) 

344 else: 

345 raise ValidationError( 

346 _('Line {line}: Invalid record extra info type "{extra_info_type}"').format( 

347 line=line_number, extra_info_type=extra_info_type 

348 ) 

349 ) 

350 

351 

352def parse_record_messages(extra_data: str) -> List[str]: 

353 msg = [] 

354 while extra_data: 

355 msg.append(extra_data[:35]) 

356 extra_data = extra_data[35:] 

357 return msg