Coverage for pystratum_common/backend/CommonRoutineLoaderWorker.py: 0%
166 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-13 08:46 +0200
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-13 08:46 +0200
1import abc
2import configparser
3import json
4import os
5from typing import Dict, List, Optional
7from pystratum_backend.RoutineLoaderWorker import RoutineLoaderWorker
8from pystratum_backend.StratumIO import StratumIO
10from pystratum_common.ConstantClass import ConstantClass
11from pystratum_common.helper.RoutineLoaderHelper import RoutineLoaderHelper
14class CommonRoutineLoaderWorker(RoutineLoaderWorker):
15 """
16 Class for loading stored routines into a RDBMS instance from (pseudo) SQL files.
17 """
19 # ------------------------------------------------------------------------------------------------------------------
20 def __init__(self, io: StratumIO, config: configparser.ConfigParser):
21 """
22 Object constructor.
24 :param io: The output decorator.
25 """
26 self.error_file_names = set()
27 """
28 A set with source names that are not loaded into RDBMS instance.
29 """
31 self._pystratum_metadata: Dict = {}
32 """
33 The meta data of all stored routines.
34 """
36 self._pystratum_metadata_filename: Optional[str] = None
37 """
38 The filename of the file with the metadata of all stored routines.
39 """
41 self._rdbms_old_metadata: Dict = {}
42 """
43 Old metadata about all stored routines.
44 """
46 self._replace_pairs: Dict = {}
47 """
48 A map from placeholders to their actual values.
49 """
51 self._source_file_encoding: Optional[str] = None
52 """
53 The character set of the source files.
54 """
56 self._source_directory: Optional[str] = None
57 """
58 Path where source files can be found.
59 """
61 self._source_file_extension: Optional[str] = None
62 """
63 The extension of the source files.
64 """
66 self._source_file_names: Dict = {}
67 """
68 All found source files.
69 """
71 self._constants_class_name: str = ''
72 """
73 The name of the class that acts like a namespace for constants.
74 """
76 self.__shadow_directory: Optional[str] = None
77 """
78 The name of the directory were copies with pure SQL of the stored routine sources must be stored.
79 """
81 self._io: StratumIO = io
82 """
83 The output decorator.
84 """
86 self._config = config
87 """
88 The configuration object.
89 """
91 # ------------------------------------------------------------------------------------------------------------------
92 def execute(self, file_names: Optional[List[str]] = None) -> int:
93 """
94 Loads stored routines into the current schema.
96 :param list[str] file_names: The sources that must be loaded. If empty all sources (if required) will loaded.
98 :rtype: int The status of exit.
99 """
100 self._io.title('Loader')
102 self._read_configuration_file()
104 if file_names:
105 self.__load_list(file_names)
106 else:
107 self.__load_all()
109 if self.error_file_names:
110 self.__log_overview_errors()
112 self._io.write_line('')
114 return 1 if self.error_file_names else 0
116 # ------------------------------------------------------------------------------------------------------------------
117 def __log_overview_errors(self) -> None:
118 """
119 Show info about sources files of stored routines that were not loaded successfully.
120 """
121 if self.error_file_names:
122 self._io.warning('Routines in the files below are not loaded:')
123 self._io.listing(sorted(self.error_file_names))
125 # ------------------------------------------------------------------------------------------------------------------
126 @abc.abstractmethod
127 def connect(self) -> None:
128 """
129 Connects to the RDBMS instance.
130 """
131 raise NotImplementedError()
133 # ------------------------------------------------------------------------------------------------------------------
134 @abc.abstractmethod
135 def disconnect(self) -> None:
136 """
137 Disconnects from the RDBMS instance.
138 """
139 raise NotImplementedError()
141 # ------------------------------------------------------------------------------------------------------------------
142 def _add_replace_pair(self, name: str, value: str, quote: bool):
143 """
144 Adds a replacement to the map of replacement pairs.
146 :param name: The name of the replacement pair.
147 :param value: The value of the replacement pair.
148 """
149 key = '@' + name + '@'
150 key = key.lower()
152 class_name = value.__class__.__name__
154 if class_name in ['int', 'float']:
155 value = str(value)
156 elif class_name in ['bool']:
157 value = '1' if value else '0'
158 elif class_name in ['str']:
159 if quote:
160 value = "'" + value + "'"
161 else:
162 self._io.log_verbose("Ignoring constant {} which is an instance of {}".format(name, class_name))
164 self._replace_pairs[key] = value
166 # ------------------------------------------------------------------------------------------------------------------
167 def __load_list(self, file_names: Optional[List[str]]) -> None:
168 """
169 Loads all stored routines in a list into the RDBMS instance.
171 :param file_names: The list of files to be loaded.
172 """
173 self.connect()
174 self.find_source_files_from_list(file_names)
175 self._get_column_type()
176 self.__read_stored_routine_metadata()
177 self.__get_constants()
178 self._get_old_stored_routine_info()
179 self._get_correct_sql_mode()
180 self.__load_stored_routines()
181 self.__write_stored_routine_metadata()
182 self.disconnect()
184 # ------------------------------------------------------------------------------------------------------------------
185 def __load_all(self) -> None:
186 """
187 Loads all stored routines into the RDBMS instance.
188 """
189 self.connect()
190 self.__find_source_files()
191 self._get_column_type()
192 self.__read_stored_routine_metadata()
193 self.__get_constants()
194 self._get_old_stored_routine_info()
195 self._get_correct_sql_mode()
196 self.__load_stored_routines()
197 self._drop_obsolete_routines()
198 self.__remove_obsolete_metadata()
199 self.__write_stored_routine_metadata()
200 self.disconnect()
202 # ------------------------------------------------------------------------------------------------------------------
203 def _read_configuration_file(self) -> None:
204 """
205 Reads parameters from the configuration file.
206 """
207 self._source_directory = self._config.get('loader', 'source_directory')
208 self._source_file_extension = self._config.get('loader', 'extension')
209 self._source_file_encoding = self._config.get('loader', 'encoding')
210 self.__shadow_directory = self._config.get('loader', 'shadow_directory', fallback=None)
211 self._pystratum_metadata_filename = self._config.get('wrapper', 'metadata')
212 self._constants_class_name = self._config.get('constants', 'class')
214 # ------------------------------------------------------------------------------------------------------------------
215 def __find_source_files(self) -> None:
216 """
217 Searches recursively for all source files in a directory.
218 """
219 for dir_path, _, files in os.walk(self._source_directory):
220 for name in files:
221 if name.lower().endswith(self._source_file_extension):
222 basename = os.path.splitext(os.path.basename(name))[0]
223 relative_path = os.path.relpath(os.path.join(dir_path, name))
225 if basename in self._source_file_names:
226 self._io.error("Files '{0}' and '{1}' have the same basename.".format(
227 self._source_file_names[basename], relative_path))
228 self.error_file_names.add(relative_path)
229 else:
230 self._source_file_names[basename] = relative_path
232 # ------------------------------------------------------------------------------------------------------------------
233 def __read_stored_routine_metadata(self) -> None:
234 """
235 Reads the metadata of stored routines from the metadata file.
236 """
237 if os.path.isfile(self._pystratum_metadata_filename):
238 with open(self._pystratum_metadata_filename, 'r') as file:
239 self._pystratum_metadata = json.load(file)
241 # ------------------------------------------------------------------------------------------------------------------
242 @abc.abstractmethod
243 def _get_column_type(self) -> None:
244 """
245 Selects schema, table, column names and the column type from the RDBMS instance and saves them as replace pairs.
246 """
247 raise NotImplementedError()
249 # ------------------------------------------------------------------------------------------------------------------
250 @abc.abstractmethod
251 def create_routine_loader_helper(self,
252 routine_name: str,
253 pystratum_old_metadata: Dict,
254 rdbms_old_metadata: Dict) -> RoutineLoaderHelper:
255 """
256 Creates a Routine Loader Helper object.
258 :param routine_name: The name of the routine.
259 :param pystratum_old_metadata: The old metadata of the stored routine from PyStratum.
260 :param rdbms_old_metadata: The old metadata of the stored routine from database instance.
261 """
262 raise NotImplementedError()
264 # ------------------------------------------------------------------------------------------------------------------
265 def __load_stored_routines(self) -> None:
266 """
267 Loads all stored routines into the RDBMS instance.
268 """
269 self._io.write_line('')
271 for routine_name in sorted(self._source_file_names):
272 if routine_name in self._pystratum_metadata:
273 old_metadata = self._pystratum_metadata[routine_name]
274 else:
275 old_metadata = None
277 if routine_name in self._rdbms_old_metadata:
278 old_routine_info = self._rdbms_old_metadata[routine_name]
279 else:
280 old_routine_info = None
282 routine_loader_helper = self.create_routine_loader_helper(routine_name, old_metadata, old_routine_info)
283 routine_loader_helper.shadow_directory = self.__shadow_directory
285 metadata = routine_loader_helper.load_stored_routine()
287 if not metadata:
288 self.error_file_names.add(self._source_file_names[routine_name])
289 if routine_name in self._pystratum_metadata:
290 del self._pystratum_metadata[routine_name]
291 else:
292 self._pystratum_metadata[routine_name] = metadata
294 # ------------------------------------------------------------------------------------------------------------------
295 @abc.abstractmethod
296 def _get_old_stored_routine_info(self) -> None:
297 """
298 Retrieves information about all stored routines in the current schema.
299 """
300 raise NotImplementedError()
302 # ------------------------------------------------------------------------------------------------------------------
303 def _get_correct_sql_mode(self) -> None:
304 """
305 Gets the SQL mode in the order as preferred by MySQL. This method is specific for MySQL.
306 """
307 pass
309 # ------------------------------------------------------------------------------------------------------------------
310 @abc.abstractmethod
311 def _drop_obsolete_routines(self) -> None:
312 """
313 Drops obsolete stored routines (i.e. stored routines that exits in the current schema but for
314 which we don't have a source file).
315 """
316 raise NotImplementedError()
318 # ------------------------------------------------------------------------------------------------------------------
319 def __remove_obsolete_metadata(self) -> None:
320 """
321 Removes obsolete entries from the metadata of all stored routines.
322 """
323 clean = {}
324 for key, _ in self._source_file_names.items():
325 if key in self._pystratum_metadata:
326 clean[key] = self._pystratum_metadata[key]
328 self._pystratum_metadata = clean
330 # ------------------------------------------------------------------------------------------------------------------
331 def __write_stored_routine_metadata(self) -> None:
332 """
333 Writes the metadata of all stored routines to the metadata file.
334 """
335 with open(self._pystratum_metadata_filename, 'w') as stream:
336 json.dump(self._pystratum_metadata, stream, indent=4, sort_keys=True)
338 # ------------------------------------------------------------------------------------------------------------------
339 def find_source_files_from_list(self, file_names) -> None:
340 """
341 Finds all source files that actually exists from a list of file names.
343 :param file_names: The list of file names.
344 """
345 for file_name in file_names:
346 if os.path.exists(file_name):
347 routine_name = os.path.splitext(os.path.basename(file_name))[0]
348 if routine_name not in self._source_file_names:
349 self._source_file_names[routine_name] = file_name
350 else:
351 self._io.error("Files '{0}' and '{1}' have the same basename.".format(
352 self._source_file_names[routine_name], file_name))
353 self.error_file_names.add(file_name)
354 else:
355 self._io.error("File not exists: '{0}'".format(file_name))
356 self.error_file_names.add(file_name)
358 # ------------------------------------------------------------------------------------------------------------------
359 def __get_constants(self) -> None:
360 """
361 Gets the constants from the class that acts like a namespace for constants and adds them to the replacement
362 pairs.
363 """
364 helper = ConstantClass(self._constants_class_name, self._io)
365 helper.reload()
366 constants = helper.constants()
368 for name, value in constants.items():
369 self._add_replace_pair(name, value, True)
371 self._io.text('Read {0} constants for substitution from <fso>{1}</fso>'.format(len(constants),
372 helper.file_name()))
374# ----------------------------------------------------------------------------------------------------------------------