Coverage for pystratum_common/helper/RoutineLoaderHelper.py: 0%
297 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-13 08:46 +0200
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-13 08:46 +0200
1import abc
2import math
3import os
4import re
5import stat
6from typing import Dict, List, Optional, Tuple, Union
8from pystratum_backend.StratumIO import StratumIO
10from pystratum_common.DocBlockReflection import DocBlockReflection
11from pystratum_common.exception.LoaderException import LoaderException
12from pystratum_common.helper.DataTypeHelper import DataTypeHelper
15class RoutineLoaderHelper(metaclass=abc.ABCMeta):
16 """
17 Class for loading a single stored routine into a RDBMS instance from a (pseudo) SQL file.
18 """
20 # ------------------------------------------------------------------------------------------------------------------
21 def __init__(self,
22 io: StratumIO,
23 routine_filename: str,
24 routine_file_encoding: str,
25 pystratum_old_metadata: Dict,
26 replace_pairs: Dict[str, str],
27 rdbms_old_metadata: Dict):
28 """
29 Object constructor.
31 :param io: The output decorator.
32 :param routine_filename: The filename of the source of the stored routine.
33 :param routine_file_encoding: The encoding of the source file.
34 :param pystratum_old_metadata: The metadata of the stored routine from PyStratum.
35 :param replace_pairs: A map from placeholders to their actual values.
36 :param rdbms_old_metadata: The old metadata of the stored routine from the RDBMS instance.
37 """
38 self._source_filename: str = routine_filename
39 """
40 The source filename holding the stored routine.
41 """
43 self._routine_file_encoding: str = routine_file_encoding
44 """
45 The encoding of the routine file.
46 """
48 self._pystratum_old_metadata: Dict = pystratum_old_metadata
49 """
50 The old metadata of the stored routine. Note: this data comes from the metadata file.
51 """
53 self._pystratum_metadata: Dict = {}
54 """
55 The metadata of the stored routine. Note: this data is stored in the metadata file and is generated by
56 pyStratum.
57 """
59 self._replace_pairs: Dict[str, str] = replace_pairs
60 """
61 A map from placeholders to their actual values.
62 """
64 self._rdbms_old_metadata: Dict = rdbms_old_metadata
65 """
66 The old information about the stored routine. Note: this data comes from the metadata of the RDBMS instance.
67 """
69 self._m_time: int = 0
70 """
71 The last modification time of the source file.
72 """
74 self._routine_name: Optional[str] = None
75 """
76 The name of the stored routine.
77 """
79 self._routine_source_code: Optional[str] = None
80 """
81 The source code as a single string of the stored routine.
82 """
84 self._routine_source_code_lines: List[str] = []
85 """
86 The source code as an array of lines string of the stored routine.
87 """
89 self._replace: Dict = {}
90 """
91 The replace pairs (i.e. placeholders and their actual values).
92 """
94 self._routine_type: Optional[str] = None
95 """
96 The stored routine type (i.e. procedure or function) of the stored routine.
97 """
99 self._designation_type: Optional[str] = None
100 """
101 The designation type of the stored routine.
102 """
104 self._doc_block_parts_source: Dict = dict()
105 """
106 All DocBlock parts as found in the source of the stored routine.
107 """
109 self._doc_block_parts_wrapper: Dict = dict()
110 """
111 The DocBlock parts to be used by the wrapper generator.
112 """
114 self._columns_types: Optional[List] = None
115 """
116 The column types of columns of the table for bulk insert of the stored routine.
117 """
119 self._fields: Optional[List] = None
120 """
121 The keys in the dictionary for bulk insert.
122 """
124 self._parameters: List[Dict] = []
125 """
126 The information about the parameters of the stored routine.
127 """
129 self._table_name: Optional[str] = None
130 """
131 If designation type is bulk_insert the table name for bulk insert.
132 """
134 self._columns: Optional[List] = None
135 """
136 The key or index columns (depending on the designation type) of the stored routine.
137 """
139 self._io: StratumIO = io
140 """
141 The output decorator.
142 """
144 self.shadow_directory: Optional[str] = None
145 """
146 The name of the directory were copies with pure SQL of the stored routine sources must be stored.
147 """
149 # ------------------------------------------------------------------------------------------------------------------
150 def load_stored_routine(self) -> Union[Dict[str, str], bool]:
151 """
152 Loads the stored routine into the instance of MySQL.
154 Returns the metadata of the stored routine if the stored routine is loaded successfully. Otherwise, returns
155 False.
156 """
157 try:
158 self._routine_name = os.path.splitext(os.path.basename(self._source_filename))[0]
160 if os.path.exists(self._source_filename):
161 if os.path.isfile(self._source_filename):
162 self._m_time = int(os.path.getmtime(self._source_filename))
163 else:
164 raise LoaderException("Unable to get mtime of file '{}'".format(self._source_filename))
165 else:
166 raise LoaderException("Source file '{}' does not exist".format(self._source_filename))
168 if self._pystratum_old_metadata:
169 self._pystratum_metadata = self._pystratum_old_metadata
171 load = self._must_reload()
172 if load:
173 self.__read_source_file()
174 self.__get_placeholders()
175 self._get_designation_type()
176 self._get_name()
177 self.__substitute_replace_pairs()
178 self._load_routine_file()
179 if self._designation_type == 'bulk_insert':
180 self._get_bulk_insert_table_columns_info()
181 self._get_routine_parameters_info()
182 self.__get_doc_block_parts_wrapper()
183 self.__save_shadow_copy()
184 self.__validate_parameter_lists()
185 self._update_metadata()
187 return self._pystratum_metadata
189 except Exception as exception:
190 self._log_exception(exception)
191 return False
193 # ------------------------------------------------------------------------------------------------------------------
194 def __validate_parameter_lists(self) -> None:
195 """
196 Validates the parameters found the DocBlock in the source of the stored routine against the parameters from the
197 metadata of MySQL and reports missing and unknown parameters names.
198 """
199 # Make list with names of parameters used in database.
200 database_parameters_names = []
201 for parameter in self._parameters:
202 database_parameters_names.append(parameter['name'])
204 # Make list with names of parameters used in dock block of routine.
205 doc_block_parameters_names = []
206 if 'parameters' in self._doc_block_parts_source:
207 for parameter in self._doc_block_parts_source['parameters']:
208 doc_block_parameters_names.append(parameter['name'])
210 # Check and show warning if any parameters is missing in doc block.
211 for parameter in database_parameters_names:
212 if parameter not in doc_block_parameters_names:
213 self._io.warning('Parameter {} is missing in doc block'.format(parameter))
215 # Check and show warning if found unknown parameters in doc block.
216 for parameter in doc_block_parameters_names:
217 if parameter not in database_parameters_names:
218 self._io.warning('Unknown parameter {} found in doc block'.format(parameter))
220 # ------------------------------------------------------------------------------------------------------------------
221 def __read_source_file(self) -> None:
222 """
223 Reads the file with the source of the stored routine.
224 """
225 with open(self._source_filename, 'r', encoding=self._routine_file_encoding) as file:
226 self._routine_source_code = file.read()
228 self._routine_source_code_lines = self._routine_source_code.split("\n")
230 # ------------------------------------------------------------------------------------------------------------------
231 def __save_shadow_copy(self) -> None:
232 """
233 Saves a copy of the stored routine source with pure SQL (if shadow directory is set).
234 """
235 if not self.shadow_directory:
236 return
238 destination_filename = os.path.join(self.shadow_directory, self._routine_name) + '.sql'
240 if os.path.realpath(destination_filename) == os.path.realpath(self._source_filename):
241 raise LoaderException("Shadow copy will override routine source '{}'".format(self._source_filename))
243 # Remove the (read only) shadow file if it exists.
244 if os.path.exists(destination_filename):
245 os.remove(destination_filename)
247 # Write the shadow file.
248 with open(destination_filename, 'wt', encoding=self._routine_file_encoding) as handle:
249 handle.write(self._routine_source_code)
251 # Make the file read only.
252 mode = os.stat(self._source_filename)[stat.ST_MODE]
253 os.chmod(destination_filename, mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH)
255 # ------------------------------------------------------------------------------------------------------------------
256 def __substitute_replace_pairs(self) -> None:
257 """
258 Substitutes all replace pairs in the source of the stored routine.
259 """
260 self._set_magic_constants()
262 routine_source = []
263 i = 0
264 for line in self._routine_source_code_lines:
265 self._replace['__LINE__'] = "'%d'" % (i + 1)
266 for search, replace in self._replace.items():
267 tmp = re.findall(search, line, re.IGNORECASE)
268 if tmp:
269 line = line.replace(tmp[0], replace)
270 routine_source.append(line)
271 i += 1
273 self._routine_source_code = "\n".join(routine_source)
275 # ------------------------------------------------------------------------------------------------------------------
276 def _log_exception(self, exception: Exception) -> None:
277 """
278 Logs an exception.
280 :param exception: The exception.
281 """
282 self._io.error(str(exception).strip().split(os.linesep))
284 # ------------------------------------------------------------------------------------------------------------------
285 @abc.abstractmethod
286 def _must_reload(self) -> bool:
287 """
288 Returns whether the source file must be load or reloaded.
289 """
290 raise NotImplementedError()
292 # ------------------------------------------------------------------------------------------------------------------
293 def __get_placeholders(self) -> None:
294 """
295 Extracts the placeholders from the stored routine source.
296 """
297 pattern = re.compile('(@[A-Za-z0-9_.]+(%(max-)?type)?@)')
298 matches = pattern.findall(self._routine_source_code)
300 placeholders = []
302 if len(matches) != 0:
303 for tmp in matches:
304 placeholder = tmp[0]
305 if placeholder.lower() not in self._replace_pairs:
306 raise LoaderException("Unknown placeholder '{0}' in file {1}".
307 format(placeholder, self._source_filename))
308 if placeholder not in placeholders:
309 placeholders.append(placeholder)
311 for placeholder in placeholders:
312 if placeholder not in self._replace:
313 self._replace[placeholder] = self._replace_pairs[placeholder.lower()]
315 # ------------------------------------------------------------------------------------------------------------------
316 def _get_designation_type(self) -> None:
317 """
318 Extracts the designation type of the stored routine.
319 """
320 self._get_designation_type_old()
321 if not self._designation_type:
322 self._get_designation_type_new()
324 # ------------------------------------------------------------------------------------------------------------------
325 def _get_designation_type_new(self) -> None:
326 """
327 Extracts the designation type of the stored routine.
328 """
329 line1, line2 = self.__get_doc_block_lines()
331 if line1 is not None and line2 is not None and line1 <= line2:
332 doc_block = self._routine_source_code_lines[line1:line2 - line1 + 1]
333 else:
334 doc_block = list()
336 reflection = DocBlockReflection(doc_block)
338 designation_type = list()
339 for tag in reflection.get_tags('type'):
340 designation_type.append(tag)
342 if len(designation_type) == 1:
343 pattern = re.compile(r'^@type\s*(\w+)\s*(.+)?\s*', re.IGNORECASE)
344 matches = pattern.findall(designation_type[0])
345 if matches:
346 self._designation_type = matches[0][0].lower()
347 tmp = str(matches[0][1])
348 if self._designation_type == 'bulk_insert':
349 n = re.compile(r'([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_,]+)', re.IGNORECASE)
350 info = n.findall(tmp)
351 if not info:
352 raise LoaderException('Expected: -- type: bulk_insert <table_name> <columns> in file {0}'.
353 format(self._source_filename))
354 self._table_name = info[0][0]
355 self._columns = str(info[0][1]).split(',')
357 elif self._designation_type == 'rows_with_key' or self._designation_type == 'rows_with_index':
358 self._columns = str(matches[0][1]).split(',')
359 else:
360 if matches[0][1]:
361 raise LoaderException('Expected: @type {}'.format(self._designation_type))
363 if not self._designation_type:
364 raise LoaderException("Unable to find the designation type of the stored routine in file {0}".
365 format(self._source_filename))
367 # ------------------------------------------------------------------------------------------------------------------
368 def _get_designation_type_old(self) -> None:
369 """
370 Extracts the designation type of the stored routine.
371 """
372 positions = self._get_specification_positions()
373 if positions[0] != -1 and positions[1] != -1:
374 pattern = re.compile(r'^\s*--\s+type\s*:\s*(\w+)\s*(.+)?\s*', re.IGNORECASE)
375 for line_number in range(positions[0], positions[1] + 1):
376 matches = pattern.findall(self._routine_source_code_lines[line_number])
377 if matches:
378 self._designation_type = matches[0][0].lower()
379 tmp = str(matches[0][1])
380 if self._designation_type == 'bulk_insert':
381 n = re.compile(r'([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_,]+)', re.IGNORECASE)
382 info = n.findall(tmp)
383 if not info:
384 raise LoaderException('Expected: -- type: bulk_insert <table_name> <columns> in file {0}'.
385 format(self._source_filename))
386 self._table_name = info[0][0]
387 self._columns = str(info[0][1]).split(',')
389 elif self._designation_type == 'rows_with_key' or self._designation_type == 'rows_with_index':
390 self._columns = str(matches[0][1]).split(',')
391 else:
392 if matches[0][1]:
393 raise LoaderException('Expected: -- type: {}'.format(self._designation_type))
395 # ------------------------------------------------------------------------------------------------------------------
396 def _get_specification_positions(self) -> Tuple[int, int]:
397 """
398 Returns a tuple with the start and end line numbers of the stored routine specification.
399 """
400 start = -1
401 for (i, line) in enumerate(self._routine_source_code_lines):
402 if self._is_start_of_stored_routine(line):
403 start = i
405 end = -1
406 for (i, line) in enumerate(self._routine_source_code_lines):
407 if self._is_start_of_stored_routine_body(line):
408 end = i - 1
410 return start, end
412 # ------------------------------------------------------------------------------------------------------------------
413 @abc.abstractmethod
414 def _is_start_of_stored_routine(self, line: str) -> bool:
415 """
416 Returns whether a line is the start of the code of the stored routine.
418 :param line: The line with source code of the stored routine.
419 """
420 raise NotImplementedError()
422 # ------------------------------------------------------------------------------------------------------------------
423 def _is_start_of_stored_routine_body(self, line: str) -> bool:
424 """
425 Returns whether a line is the start of the body of the stored routine.
427 :param line: The line with source code of the stored routine.
428 """
429 raise NotImplementedError()
431 # ------------------------------------------------------------------------------------------------------------------
432 def __get_doc_block_lines(self) -> Tuple[int, int]:
433 """
434 Returns the start and end line of the DocBlock of the stored routine code.
435 """
436 line1 = None
437 line2 = None
439 i = 0
440 for line in self._routine_source_code_lines:
441 if re.match(r'\s*/\*\*', line):
442 line1 = i
444 if re.match(r'\s*\*/', line):
445 line2 = i
447 if self._is_start_of_stored_routine(line):
448 break
450 i += 1
452 return line1, line2
454 # ------------------------------------------------------------------------------------------------------------------
455 def __get_doc_block_parts_source(self) -> None:
456 """
457 Extracts the DocBlock (in parts) from the source of the stored routine source.
458 """
459 line1, line2 = self.__get_doc_block_lines()
461 if line1 is not None and line2 is not None and line1 <= line2:
462 doc_block = self._routine_source_code_lines[line1:line2 - line1 + 1]
463 else:
464 doc_block = list()
466 reflection = DocBlockReflection(doc_block)
468 self._doc_block_parts_source['description'] = reflection.get_description()
470 self._doc_block_parts_source['parameters'] = list()
471 for tag in reflection.get_tags('param'):
472 parts = re.match(r'^(@param)\s+(\w+)\s*(.+)?', tag, re.DOTALL)
473 if parts:
474 self._doc_block_parts_source['parameters'].append({'name': parts.group(2),
475 'description': parts.group(3)})
477 # ------------------------------------------------------------------------------------------------------------------
478 def __get_parameter_doc_description(self, name: str) -> str:
479 """
480 Returns the description by name of the parameter as found in the DocBlock of the stored routine.
482 :param name: The name of the parameter.
483 """
484 for param in self._doc_block_parts_source['parameters']:
485 if param['name'] == name:
486 return param['description']
488 return ''
490 # ------------------------------------------------------------------------------------------------------------------
491 @abc.abstractmethod
492 def _get_data_type_helper(self) -> DataTypeHelper:
493 """
494 Returns a data type helper object appropriate for the RDBMS.
495 """
496 raise NotImplementedError()
498 # ------------------------------------------------------------------------------------------------------------------
499 def __get_doc_block_parts_wrapper(self) -> None:
500 """
501 Generates the DocBlock parts to be used by the wrapper generator.
502 """
503 self.__get_doc_block_parts_source()
505 helper = self._get_data_type_helper()
507 parameters = list()
508 for parameter_info in self._parameters:
509 parameters.append(
510 {'parameter_name': parameter_info['name'],
511 'python_type': helper.column_type_to_python_type(parameter_info),
512 'python_type_hint': helper.column_type_to_python_type_hint(parameter_info),
513 'data_type_descriptor': parameter_info['data_type_descriptor'],
514 'description': self.__get_parameter_doc_description(parameter_info['name'])})
516 self._doc_block_parts_wrapper['description'] = self._doc_block_parts_source['description']
517 self._doc_block_parts_wrapper['parameters'] = parameters
519 # ------------------------------------------------------------------------------------------------------------------
520 @abc.abstractmethod
521 def _get_name(self) -> None:
522 """
523 Extracts the name of the stored routine and the stored routine type (i.e. procedure or function) source.
524 """
525 raise NotImplementedError()
527 # ------------------------------------------------------------------------------------------------------------------
528 @abc.abstractmethod
529 def _load_routine_file(self) -> None:
530 """
531 Loads the stored routine into the RDBMS instance.
532 """
533 raise NotImplementedError()
535 # ------------------------------------------------------------------------------------------------------------------
536 @abc.abstractmethod
537 def _get_bulk_insert_table_columns_info(self) -> None:
538 """
539 Gets the column names and column types of the current table for bulk insert.
540 """
541 raise NotImplementedError()
543 # ------------------------------------------------------------------------------------------------------------------
544 @abc.abstractmethod
545 def _get_routine_parameters_info(self) -> None:
546 """
547 Retrieves information about the stored routine parameters from the metadata of the RDBMS.
548 """
549 raise NotImplementedError()
551 # ------------------------------------------------------------------------------------------------------------------
552 def _update_metadata(self) -> None:
553 """
554 Updates the metadata of the stored routine.
555 """
556 self._pystratum_metadata['routine_name'] = self._routine_name
557 self._pystratum_metadata['designation'] = self._designation_type
558 self._pystratum_metadata['table_name'] = self._table_name
559 self._pystratum_metadata['parameters'] = self._parameters
560 self._pystratum_metadata['columns'] = self._columns
561 self._pystratum_metadata['fields'] = self._fields
562 self._pystratum_metadata['column_types'] = self._columns_types
563 self._pystratum_metadata['timestamp'] = self._m_time
564 self._pystratum_metadata['replace'] = self._replace
565 self._pystratum_metadata['pydoc'] = self._doc_block_parts_wrapper
567 # ------------------------------------------------------------------------------------------------------------------
568 @abc.abstractmethod
569 def _drop_routine(self) -> None:
570 """
571 Drops the stored routine if it exists.
572 """
573 raise NotImplementedError()
575 # ------------------------------------------------------------------------------------------------------------------
576 def _set_magic_constants(self) -> None:
577 """
578 Adds magic constants to replace list.
579 """
580 real_path = os.path.realpath(self._source_filename)
582 self._replace['__FILE__'] = "'%s'" % real_path
583 self._replace['__ROUTINE__'] = "'%s'" % self._routine_name
584 self._replace['__DIR__'] = "'%s'" % os.path.dirname(real_path)
586 # ------------------------------------------------------------------------------------------------------------------
587 def _unset_magic_constants(self) -> None:
588 """
589 Removes magic constants from current replace list.
590 """
591 if '__FILE__' in self._replace:
592 del self._replace['__FILE__']
594 if '__ROUTINE__' in self._replace:
595 del self._replace['__ROUTINE__']
597 if '__DIR__' in self._replace:
598 del self._replace['__DIR__']
600 if '__LINE__' in self._replace:
601 del self._replace['__LINE__']
603 # ------------------------------------------------------------------------------------------------------------------
604 def _print_sql_with_error(self, sql: str, error_line: int) -> None:
605 """
606 Writes a SQL statement with a syntax error to the output. The line where the error occurs is highlighted.
608 :param sql: The SQL statement.
609 :param error_line: The line where the error occurs.
610 """
611 if os.linesep in sql:
612 lines = sql.split(os.linesep)
613 digits = math.ceil(math.log(len(lines) + 1, 10))
614 i = 1
615 for line in lines:
616 if i == error_line:
617 self._io.text('<error>{0:{width}} {1}</error>'.format(i, line, width=digits, ))
618 else:
619 self._io.text('{0:{width}} {1}'.format(i, line, width=digits, ))
620 i += 1
621 else:
622 self._io.text(sql)
624# ----------------------------------------------------------------------------------------------------------------------