Coverage for pystratum_common/helper/RoutineLoaderHelper.py: 0%

297 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-13 08:46 +0200

1import abc 

2import math 

3import os 

4import re 

5import stat 

6from typing import Dict, List, Optional, Tuple, Union 

7 

8from pystratum_backend.StratumIO import StratumIO 

9 

10from pystratum_common.DocBlockReflection import DocBlockReflection 

11from pystratum_common.exception.LoaderException import LoaderException 

12from pystratum_common.helper.DataTypeHelper import DataTypeHelper 

13 

14 

15class RoutineLoaderHelper(metaclass=abc.ABCMeta): 

16 """ 

17 Class for loading a single stored routine into a RDBMS instance from a (pseudo) SQL file. 

18 """ 

19 

20 # ------------------------------------------------------------------------------------------------------------------ 

21 def __init__(self, 

22 io: StratumIO, 

23 routine_filename: str, 

24 routine_file_encoding: str, 

25 pystratum_old_metadata: Dict, 

26 replace_pairs: Dict[str, str], 

27 rdbms_old_metadata: Dict): 

28 """ 

29 Object constructor. 

30 

31 :param io: The output decorator. 

32 :param routine_filename: The filename of the source of the stored routine. 

33 :param routine_file_encoding: The encoding of the source file. 

34 :param pystratum_old_metadata: The metadata of the stored routine from PyStratum. 

35 :param replace_pairs: A map from placeholders to their actual values. 

36 :param rdbms_old_metadata: The old metadata of the stored routine from the RDBMS instance. 

37 """ 

38 self._source_filename: str = routine_filename 

39 """ 

40 The source filename holding the stored routine. 

41 """ 

42 

43 self._routine_file_encoding: str = routine_file_encoding 

44 """ 

45 The encoding of the routine file. 

46 """ 

47 

48 self._pystratum_old_metadata: Dict = pystratum_old_metadata 

49 """ 

50 The old metadata of the stored routine. Note: this data comes from the metadata file. 

51 """ 

52 

53 self._pystratum_metadata: Dict = {} 

54 """ 

55 The metadata of the stored routine. Note: this data is stored in the metadata file and is generated by 

56 pyStratum. 

57 """ 

58 

59 self._replace_pairs: Dict[str, str] = replace_pairs 

60 """ 

61 A map from placeholders to their actual values. 

62 """ 

63 

64 self._rdbms_old_metadata: Dict = rdbms_old_metadata 

65 """ 

66 The old information about the stored routine. Note: this data comes from the metadata of the RDBMS instance. 

67 """ 

68 

69 self._m_time: int = 0 

70 """ 

71 The last modification time of the source file. 

72 """ 

73 

74 self._routine_name: Optional[str] = None 

75 """ 

76 The name of the stored routine. 

77 """ 

78 

79 self._routine_source_code: Optional[str] = None 

80 """ 

81 The source code as a single string of the stored routine. 

82 """ 

83 

84 self._routine_source_code_lines: List[str] = [] 

85 """ 

86 The source code as an array of lines string of the stored routine. 

87 """ 

88 

89 self._replace: Dict = {} 

90 """ 

91 The replace pairs (i.e. placeholders and their actual values). 

92 """ 

93 

94 self._routine_type: Optional[str] = None 

95 """ 

96 The stored routine type (i.e. procedure or function) of the stored routine. 

97 """ 

98 

99 self._designation_type: Optional[str] = None 

100 """ 

101 The designation type of the stored routine. 

102 """ 

103 

104 self._doc_block_parts_source: Dict = dict() 

105 """ 

106 All DocBlock parts as found in the source of the stored routine. 

107 """ 

108 

109 self._doc_block_parts_wrapper: Dict = dict() 

110 """ 

111 The DocBlock parts to be used by the wrapper generator. 

112 """ 

113 

114 self._columns_types: Optional[List] = None 

115 """ 

116 The column types of columns of the table for bulk insert of the stored routine. 

117 """ 

118 

119 self._fields: Optional[List] = None 

120 """ 

121 The keys in the dictionary for bulk insert. 

122 """ 

123 

124 self._parameters: List[Dict] = [] 

125 """ 

126 The information about the parameters of the stored routine. 

127 """ 

128 

129 self._table_name: Optional[str] = None 

130 """ 

131 If designation type is bulk_insert the table name for bulk insert. 

132 """ 

133 

134 self._columns: Optional[List] = None 

135 """ 

136 The key or index columns (depending on the designation type) of the stored routine. 

137 """ 

138 

139 self._io: StratumIO = io 

140 """ 

141 The output decorator. 

142 """ 

143 

144 self.shadow_directory: Optional[str] = None 

145 """ 

146 The name of the directory were copies with pure SQL of the stored routine sources must be stored. 

147 """ 

148 

149 # ------------------------------------------------------------------------------------------------------------------ 

150 def load_stored_routine(self) -> Union[Dict[str, str], bool]: 

151 """ 

152 Loads the stored routine into the instance of MySQL. 

153 

154 Returns the metadata of the stored routine if the stored routine is loaded successfully. Otherwise, returns 

155 False. 

156 """ 

157 try: 

158 self._routine_name = os.path.splitext(os.path.basename(self._source_filename))[0] 

159 

160 if os.path.exists(self._source_filename): 

161 if os.path.isfile(self._source_filename): 

162 self._m_time = int(os.path.getmtime(self._source_filename)) 

163 else: 

164 raise LoaderException("Unable to get mtime of file '{}'".format(self._source_filename)) 

165 else: 

166 raise LoaderException("Source file '{}' does not exist".format(self._source_filename)) 

167 

168 if self._pystratum_old_metadata: 

169 self._pystratum_metadata = self._pystratum_old_metadata 

170 

171 load = self._must_reload() 

172 if load: 

173 self.__read_source_file() 

174 self.__get_placeholders() 

175 self._get_designation_type() 

176 self._get_name() 

177 self.__substitute_replace_pairs() 

178 self._load_routine_file() 

179 if self._designation_type == 'bulk_insert': 

180 self._get_bulk_insert_table_columns_info() 

181 self._get_routine_parameters_info() 

182 self.__get_doc_block_parts_wrapper() 

183 self.__save_shadow_copy() 

184 self.__validate_parameter_lists() 

185 self._update_metadata() 

186 

187 return self._pystratum_metadata 

188 

189 except Exception as exception: 

190 self._log_exception(exception) 

191 return False 

192 

193 # ------------------------------------------------------------------------------------------------------------------ 

194 def __validate_parameter_lists(self) -> None: 

195 """ 

196 Validates the parameters found the DocBlock in the source of the stored routine against the parameters from the 

197 metadata of MySQL and reports missing and unknown parameters names. 

198 """ 

199 # Make list with names of parameters used in database. 

200 database_parameters_names = [] 

201 for parameter in self._parameters: 

202 database_parameters_names.append(parameter['name']) 

203 

204 # Make list with names of parameters used in dock block of routine. 

205 doc_block_parameters_names = [] 

206 if 'parameters' in self._doc_block_parts_source: 

207 for parameter in self._doc_block_parts_source['parameters']: 

208 doc_block_parameters_names.append(parameter['name']) 

209 

210 # Check and show warning if any parameters is missing in doc block. 

211 for parameter in database_parameters_names: 

212 if parameter not in doc_block_parameters_names: 

213 self._io.warning('Parameter {} is missing in doc block'.format(parameter)) 

214 

215 # Check and show warning if found unknown parameters in doc block. 

216 for parameter in doc_block_parameters_names: 

217 if parameter not in database_parameters_names: 

218 self._io.warning('Unknown parameter {} found in doc block'.format(parameter)) 

219 

220 # ------------------------------------------------------------------------------------------------------------------ 

221 def __read_source_file(self) -> None: 

222 """ 

223 Reads the file with the source of the stored routine. 

224 """ 

225 with open(self._source_filename, 'r', encoding=self._routine_file_encoding) as file: 

226 self._routine_source_code = file.read() 

227 

228 self._routine_source_code_lines = self._routine_source_code.split("\n") 

229 

230 # ------------------------------------------------------------------------------------------------------------------ 

231 def __save_shadow_copy(self) -> None: 

232 """ 

233 Saves a copy of the stored routine source with pure SQL (if shadow directory is set). 

234 """ 

235 if not self.shadow_directory: 

236 return 

237 

238 destination_filename = os.path.join(self.shadow_directory, self._routine_name) + '.sql' 

239 

240 if os.path.realpath(destination_filename) == os.path.realpath(self._source_filename): 

241 raise LoaderException("Shadow copy will override routine source '{}'".format(self._source_filename)) 

242 

243 # Remove the (read only) shadow file if it exists. 

244 if os.path.exists(destination_filename): 

245 os.remove(destination_filename) 

246 

247 # Write the shadow file. 

248 with open(destination_filename, 'wt', encoding=self._routine_file_encoding) as handle: 

249 handle.write(self._routine_source_code) 

250 

251 # Make the file read only. 

252 mode = os.stat(self._source_filename)[stat.ST_MODE] 

253 os.chmod(destination_filename, mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH) 

254 

255 # ------------------------------------------------------------------------------------------------------------------ 

256 def __substitute_replace_pairs(self) -> None: 

257 """ 

258 Substitutes all replace pairs in the source of the stored routine. 

259 """ 

260 self._set_magic_constants() 

261 

262 routine_source = [] 

263 i = 0 

264 for line in self._routine_source_code_lines: 

265 self._replace['__LINE__'] = "'%d'" % (i + 1) 

266 for search, replace in self._replace.items(): 

267 tmp = re.findall(search, line, re.IGNORECASE) 

268 if tmp: 

269 line = line.replace(tmp[0], replace) 

270 routine_source.append(line) 

271 i += 1 

272 

273 self._routine_source_code = "\n".join(routine_source) 

274 

275 # ------------------------------------------------------------------------------------------------------------------ 

276 def _log_exception(self, exception: Exception) -> None: 

277 """ 

278 Logs an exception. 

279 

280 :param exception: The exception. 

281 """ 

282 self._io.error(str(exception).strip().split(os.linesep)) 

283 

284 # ------------------------------------------------------------------------------------------------------------------ 

285 @abc.abstractmethod 

286 def _must_reload(self) -> bool: 

287 """ 

288 Returns whether the source file must be load or reloaded. 

289 """ 

290 raise NotImplementedError() 

291 

292 # ------------------------------------------------------------------------------------------------------------------ 

293 def __get_placeholders(self) -> None: 

294 """ 

295 Extracts the placeholders from the stored routine source. 

296 """ 

297 pattern = re.compile('(@[A-Za-z0-9_.]+(%(max-)?type)?@)') 

298 matches = pattern.findall(self._routine_source_code) 

299 

300 placeholders = [] 

301 

302 if len(matches) != 0: 

303 for tmp in matches: 

304 placeholder = tmp[0] 

305 if placeholder.lower() not in self._replace_pairs: 

306 raise LoaderException("Unknown placeholder '{0}' in file {1}". 

307 format(placeholder, self._source_filename)) 

308 if placeholder not in placeholders: 

309 placeholders.append(placeholder) 

310 

311 for placeholder in placeholders: 

312 if placeholder not in self._replace: 

313 self._replace[placeholder] = self._replace_pairs[placeholder.lower()] 

314 

315 # ------------------------------------------------------------------------------------------------------------------ 

316 def _get_designation_type(self) -> None: 

317 """ 

318 Extracts the designation type of the stored routine. 

319 """ 

320 self._get_designation_type_old() 

321 if not self._designation_type: 

322 self._get_designation_type_new() 

323 

324 # ------------------------------------------------------------------------------------------------------------------ 

325 def _get_designation_type_new(self) -> None: 

326 """ 

327 Extracts the designation type of the stored routine. 

328 """ 

329 line1, line2 = self.__get_doc_block_lines() 

330 

331 if line1 is not None and line2 is not None and line1 <= line2: 

332 doc_block = self._routine_source_code_lines[line1:line2 - line1 + 1] 

333 else: 

334 doc_block = list() 

335 

336 reflection = DocBlockReflection(doc_block) 

337 

338 designation_type = list() 

339 for tag in reflection.get_tags('type'): 

340 designation_type.append(tag) 

341 

342 if len(designation_type) == 1: 

343 pattern = re.compile(r'^@type\s*(\w+)\s*(.+)?\s*', re.IGNORECASE) 

344 matches = pattern.findall(designation_type[0]) 

345 if matches: 

346 self._designation_type = matches[0][0].lower() 

347 tmp = str(matches[0][1]) 

348 if self._designation_type == 'bulk_insert': 

349 n = re.compile(r'([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_,]+)', re.IGNORECASE) 

350 info = n.findall(tmp) 

351 if not info: 

352 raise LoaderException('Expected: -- type: bulk_insert <table_name> <columns> in file {0}'. 

353 format(self._source_filename)) 

354 self._table_name = info[0][0] 

355 self._columns = str(info[0][1]).split(',') 

356 

357 elif self._designation_type == 'rows_with_key' or self._designation_type == 'rows_with_index': 

358 self._columns = str(matches[0][1]).split(',') 

359 else: 

360 if matches[0][1]: 

361 raise LoaderException('Expected: @type {}'.format(self._designation_type)) 

362 

363 if not self._designation_type: 

364 raise LoaderException("Unable to find the designation type of the stored routine in file {0}". 

365 format(self._source_filename)) 

366 

367 # ------------------------------------------------------------------------------------------------------------------ 

368 def _get_designation_type_old(self) -> None: 

369 """ 

370 Extracts the designation type of the stored routine. 

371 """ 

372 positions = self._get_specification_positions() 

373 if positions[0] != -1 and positions[1] != -1: 

374 pattern = re.compile(r'^\s*--\s+type\s*:\s*(\w+)\s*(.+)?\s*', re.IGNORECASE) 

375 for line_number in range(positions[0], positions[1] + 1): 

376 matches = pattern.findall(self._routine_source_code_lines[line_number]) 

377 if matches: 

378 self._designation_type = matches[0][0].lower() 

379 tmp = str(matches[0][1]) 

380 if self._designation_type == 'bulk_insert': 

381 n = re.compile(r'([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_,]+)', re.IGNORECASE) 

382 info = n.findall(tmp) 

383 if not info: 

384 raise LoaderException('Expected: -- type: bulk_insert <table_name> <columns> in file {0}'. 

385 format(self._source_filename)) 

386 self._table_name = info[0][0] 

387 self._columns = str(info[0][1]).split(',') 

388 

389 elif self._designation_type == 'rows_with_key' or self._designation_type == 'rows_with_index': 

390 self._columns = str(matches[0][1]).split(',') 

391 else: 

392 if matches[0][1]: 

393 raise LoaderException('Expected: -- type: {}'.format(self._designation_type)) 

394 

395 # ------------------------------------------------------------------------------------------------------------------ 

396 def _get_specification_positions(self) -> Tuple[int, int]: 

397 """ 

398 Returns a tuple with the start and end line numbers of the stored routine specification. 

399 """ 

400 start = -1 

401 for (i, line) in enumerate(self._routine_source_code_lines): 

402 if self._is_start_of_stored_routine(line): 

403 start = i 

404 

405 end = -1 

406 for (i, line) in enumerate(self._routine_source_code_lines): 

407 if self._is_start_of_stored_routine_body(line): 

408 end = i - 1 

409 

410 return start, end 

411 

412 # ------------------------------------------------------------------------------------------------------------------ 

413 @abc.abstractmethod 

414 def _is_start_of_stored_routine(self, line: str) -> bool: 

415 """ 

416 Returns whether a line is the start of the code of the stored routine. 

417 

418 :param line: The line with source code of the stored routine. 

419 """ 

420 raise NotImplementedError() 

421 

422 # ------------------------------------------------------------------------------------------------------------------ 

423 def _is_start_of_stored_routine_body(self, line: str) -> bool: 

424 """ 

425 Returns whether a line is the start of the body of the stored routine. 

426 

427 :param line: The line with source code of the stored routine. 

428 """ 

429 raise NotImplementedError() 

430 

431 # ------------------------------------------------------------------------------------------------------------------ 

432 def __get_doc_block_lines(self) -> Tuple[int, int]: 

433 """ 

434 Returns the start and end line of the DocBlock of the stored routine code. 

435 """ 

436 line1 = None 

437 line2 = None 

438 

439 i = 0 

440 for line in self._routine_source_code_lines: 

441 if re.match(r'\s*/\*\*', line): 

442 line1 = i 

443 

444 if re.match(r'\s*\*/', line): 

445 line2 = i 

446 

447 if self._is_start_of_stored_routine(line): 

448 break 

449 

450 i += 1 

451 

452 return line1, line2 

453 

454 # ------------------------------------------------------------------------------------------------------------------ 

455 def __get_doc_block_parts_source(self) -> None: 

456 """ 

457 Extracts the DocBlock (in parts) from the source of the stored routine source. 

458 """ 

459 line1, line2 = self.__get_doc_block_lines() 

460 

461 if line1 is not None and line2 is not None and line1 <= line2: 

462 doc_block = self._routine_source_code_lines[line1:line2 - line1 + 1] 

463 else: 

464 doc_block = list() 

465 

466 reflection = DocBlockReflection(doc_block) 

467 

468 self._doc_block_parts_source['description'] = reflection.get_description() 

469 

470 self._doc_block_parts_source['parameters'] = list() 

471 for tag in reflection.get_tags('param'): 

472 parts = re.match(r'^(@param)\s+(\w+)\s*(.+)?', tag, re.DOTALL) 

473 if parts: 

474 self._doc_block_parts_source['parameters'].append({'name': parts.group(2), 

475 'description': parts.group(3)}) 

476 

477 # ------------------------------------------------------------------------------------------------------------------ 

478 def __get_parameter_doc_description(self, name: str) -> str: 

479 """ 

480 Returns the description by name of the parameter as found in the DocBlock of the stored routine. 

481 

482 :param name: The name of the parameter. 

483 """ 

484 for param in self._doc_block_parts_source['parameters']: 

485 if param['name'] == name: 

486 return param['description'] 

487 

488 return '' 

489 

490 # ------------------------------------------------------------------------------------------------------------------ 

491 @abc.abstractmethod 

492 def _get_data_type_helper(self) -> DataTypeHelper: 

493 """ 

494 Returns a data type helper object appropriate for the RDBMS. 

495 """ 

496 raise NotImplementedError() 

497 

498 # ------------------------------------------------------------------------------------------------------------------ 

499 def __get_doc_block_parts_wrapper(self) -> None: 

500 """ 

501 Generates the DocBlock parts to be used by the wrapper generator. 

502 """ 

503 self.__get_doc_block_parts_source() 

504 

505 helper = self._get_data_type_helper() 

506 

507 parameters = list() 

508 for parameter_info in self._parameters: 

509 parameters.append( 

510 {'parameter_name': parameter_info['name'], 

511 'python_type': helper.column_type_to_python_type(parameter_info), 

512 'python_type_hint': helper.column_type_to_python_type_hint(parameter_info), 

513 'data_type_descriptor': parameter_info['data_type_descriptor'], 

514 'description': self.__get_parameter_doc_description(parameter_info['name'])}) 

515 

516 self._doc_block_parts_wrapper['description'] = self._doc_block_parts_source['description'] 

517 self._doc_block_parts_wrapper['parameters'] = parameters 

518 

519 # ------------------------------------------------------------------------------------------------------------------ 

520 @abc.abstractmethod 

521 def _get_name(self) -> None: 

522 """ 

523 Extracts the name of the stored routine and the stored routine type (i.e. procedure or function) source. 

524 """ 

525 raise NotImplementedError() 

526 

527 # ------------------------------------------------------------------------------------------------------------------ 

528 @abc.abstractmethod 

529 def _load_routine_file(self) -> None: 

530 """ 

531 Loads the stored routine into the RDBMS instance. 

532 """ 

533 raise NotImplementedError() 

534 

535 # ------------------------------------------------------------------------------------------------------------------ 

536 @abc.abstractmethod 

537 def _get_bulk_insert_table_columns_info(self) -> None: 

538 """ 

539 Gets the column names and column types of the current table for bulk insert. 

540 """ 

541 raise NotImplementedError() 

542 

543 # ------------------------------------------------------------------------------------------------------------------ 

544 @abc.abstractmethod 

545 def _get_routine_parameters_info(self) -> None: 

546 """ 

547 Retrieves information about the stored routine parameters from the metadata of the RDBMS. 

548 """ 

549 raise NotImplementedError() 

550 

551 # ------------------------------------------------------------------------------------------------------------------ 

552 def _update_metadata(self) -> None: 

553 """ 

554 Updates the metadata of the stored routine. 

555 """ 

556 self._pystratum_metadata['routine_name'] = self._routine_name 

557 self._pystratum_metadata['designation'] = self._designation_type 

558 self._pystratum_metadata['table_name'] = self._table_name 

559 self._pystratum_metadata['parameters'] = self._parameters 

560 self._pystratum_metadata['columns'] = self._columns 

561 self._pystratum_metadata['fields'] = self._fields 

562 self._pystratum_metadata['column_types'] = self._columns_types 

563 self._pystratum_metadata['timestamp'] = self._m_time 

564 self._pystratum_metadata['replace'] = self._replace 

565 self._pystratum_metadata['pydoc'] = self._doc_block_parts_wrapper 

566 

567 # ------------------------------------------------------------------------------------------------------------------ 

568 @abc.abstractmethod 

569 def _drop_routine(self) -> None: 

570 """ 

571 Drops the stored routine if it exists. 

572 """ 

573 raise NotImplementedError() 

574 

575 # ------------------------------------------------------------------------------------------------------------------ 

576 def _set_magic_constants(self) -> None: 

577 """ 

578 Adds magic constants to replace list. 

579 """ 

580 real_path = os.path.realpath(self._source_filename) 

581 

582 self._replace['__FILE__'] = "'%s'" % real_path 

583 self._replace['__ROUTINE__'] = "'%s'" % self._routine_name 

584 self._replace['__DIR__'] = "'%s'" % os.path.dirname(real_path) 

585 

586 # ------------------------------------------------------------------------------------------------------------------ 

587 def _unset_magic_constants(self) -> None: 

588 """ 

589 Removes magic constants from current replace list. 

590 """ 

591 if '__FILE__' in self._replace: 

592 del self._replace['__FILE__'] 

593 

594 if '__ROUTINE__' in self._replace: 

595 del self._replace['__ROUTINE__'] 

596 

597 if '__DIR__' in self._replace: 

598 del self._replace['__DIR__'] 

599 

600 if '__LINE__' in self._replace: 

601 del self._replace['__LINE__'] 

602 

603 # ------------------------------------------------------------------------------------------------------------------ 

604 def _print_sql_with_error(self, sql: str, error_line: int) -> None: 

605 """ 

606 Writes a SQL statement with a syntax error to the output. The line where the error occurs is highlighted. 

607 

608 :param sql: The SQL statement. 

609 :param error_line: The line where the error occurs. 

610 """ 

611 if os.linesep in sql: 

612 lines = sql.split(os.linesep) 

613 digits = math.ceil(math.log(len(lines) + 1, 10)) 

614 i = 1 

615 for line in lines: 

616 if i == error_line: 

617 self._io.text('<error>{0:{width}} {1}</error>'.format(i, line, width=digits, )) 

618 else: 

619 self._io.text('{0:{width}} {1}'.format(i, line, width=digits, )) 

620 i += 1 

621 else: 

622 self._io.text(sql) 

623 

624# ----------------------------------------------------------------------------------------------------------------------