Coverage for pystratum_common/backend/CommonRoutineLoaderWorker.py : 0%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import abc
2import configparser
3import json
4import os
5from typing import Dict, List, Optional
7from pystratum_backend.RoutineLoaderWorker import RoutineLoaderWorker
8from pystratum_backend.StratumStyle import StratumStyle
9from pystratum_common.ConstantClass import ConstantClass
10from pystratum_common.helper.RoutineLoaderHelper import RoutineLoaderHelper
13class CommonRoutineLoaderWorker(RoutineLoaderWorker):
14 """
15 Class for loading stored routines into a RDBMS instance from (pseudo) SQL files.
16 """
18 # ------------------------------------------------------------------------------------------------------------------
19 def __init__(self, io: StratumStyle, config: configparser.ConfigParser):
20 """
21 Object constructor.
23 :param pystratum.style.PyStratumStyle.PyStratumStyle io: The output decorator.
24 """
25 self.error_file_names = set()
26 """
27 A set with source names that are not loaded into RDBMS instance.
29 :type: set
30 """
32 self._pystratum_metadata: Dict = {}
33 """
34 The meta data of all stored routines.
35 """
37 self._pystratum_metadata_filename: Optional[str] = None
38 """
39 The filename of the file with the metadata of all stored routines.
40 """
42 self._rdbms_old_metadata: Dict = {}
43 """
44 Old metadata about all stored routines.
45 """
47 self._replace_pairs: Dict = {}
48 """
49 A map from placeholders to their actual values.
50 """
52 self._source_file_encoding: Optional[str] = None
53 """
54 The character set of the source files.
55 """
57 self._source_directory: Optional[str] = None
58 """
59 Path where source files can be found.
60 """
62 self._source_file_extension: Optional[str] = None
63 """
64 The extension of the source files.
65 """
67 self._source_file_names: Dict = {}
68 """
69 All found source files.
70 """
72 self._constants_class_name: str = ''
73 """
74 The name of the class that acts like a namespace for constants.
75 """
77 self.__shadow_directory: Optional[str] = None
78 """
79 The name of the directory were copies with pure SQL of the stored routine sources must be stored.
80 """
82 self._io: StratumStyle = io
83 """
84 The output decorator.
85 """
87 self._config = config
88 """
89 The configuration object.
91 :type: ConfigParser
92 """
94 # ------------------------------------------------------------------------------------------------------------------
95 def execute(self, file_names: Optional[List[str]] = None) -> int:
96 """
97 Loads stored routines into the current schema.
99 :param list[str] file_names: The sources that must be loaded. If empty all sources (if required) will loaded.
101 :rtype: int The status of exit.
102 """
103 self._io.title('Loader')
105 self._read_configuration_file()
107 if file_names:
108 self.__load_list(file_names)
109 else:
110 self.__load_all()
112 if self.error_file_names:
113 self.__log_overview_errors()
115 self._io.writeln('')
117 return 1 if self.error_file_names else 0
119 # ------------------------------------------------------------------------------------------------------------------
120 def __log_overview_errors(self) -> None:
121 """
122 Show info about sources files of stored routines that were not loaded successfully.
123 """
124 if self.error_file_names:
125 self._io.warning('Routines in the files below are not loaded:')
126 self._io.listing(sorted(self.error_file_names))
128 # ------------------------------------------------------------------------------------------------------------------
129 @abc.abstractmethod
130 def connect(self) -> None:
131 """
132 Connects to the RDBMS instance.
133 """
134 raise NotImplementedError()
136 # ------------------------------------------------------------------------------------------------------------------
137 @abc.abstractmethod
138 def disconnect(self) -> None:
139 """
140 Disconnects from the RDBMS instance.
141 """
142 raise NotImplementedError()
144 # ------------------------------------------------------------------------------------------------------------------
145 def _add_replace_pair(self, name: str, value: str, quote: bool):
146 """
147 Adds a replace part to the map of replace pairs.
149 :param name: The name of the replace pair.
150 :param value: The value of value of the replace pair.
151 """
152 key = '@' + name + '@'
153 key = key.lower()
155 class_name = value.__class__.__name__
157 if class_name in ['int', 'float']:
158 value = str(value)
159 elif class_name in ['bool']:
160 value = '1' if value else '0'
161 elif class_name in ['str']:
162 if quote:
163 value = "'" + value + "'"
164 else:
165 self._io.log_verbose("Ignoring constant {} which is an instance of {}".format(name, class_name))
167 self._replace_pairs[key] = value
169 # ------------------------------------------------------------------------------------------------------------------
170 def __load_list(self, file_names: Optional[List[str]]) -> None:
171 """
172 Loads all stored routines in a list into the RDBMS instance.
173 :param list[str] file_names: The list of files to be loaded.
174 """
175 self.connect()
176 self.find_source_files_from_list(file_names)
177 self._get_column_type()
178 self.__read_stored_routine_metadata()
179 self.__get_constants()
180 self._get_old_stored_routine_info()
181 self._get_correct_sql_mode()
182 self.__load_stored_routines()
183 self.__write_stored_routine_metadata()
184 self.disconnect()
186 # ------------------------------------------------------------------------------------------------------------------
187 def __load_all(self) -> None:
188 """
189 Loads all stored routines into the RDBMS instance.
190 """
191 self.connect()
192 self.__find_source_files()
193 self._get_column_type()
194 self.__read_stored_routine_metadata()
195 self.__get_constants()
196 self._get_old_stored_routine_info()
197 self._get_correct_sql_mode()
198 self.__load_stored_routines()
199 self._drop_obsolete_routines()
200 self.__remove_obsolete_metadata()
201 self.__write_stored_routine_metadata()
202 self.disconnect()
204 # ------------------------------------------------------------------------------------------------------------------
205 def _read_configuration_file(self) -> None:
206 """
207 Reads parameters from the configuration file.
208 """
209 self._source_directory = self._config.get('loader', 'source_directory')
210 self._source_file_extension = self._config.get('loader', 'extension')
211 self._source_file_encoding = self._config.get('loader', 'encoding')
212 self.__shadow_directory = self._config.get('loader', 'shadow_directory', fallback=None)
214 self._pystratum_metadata_filename = self._config.get('wrapper', 'metadata')
216 self._constants_class_name = self._config.get('constants', 'class')
218 # ------------------------------------------------------------------------------------------------------------------
219 def __find_source_files(self) -> None:
220 """
221 Searches recursively for all source files in a directory.
222 """
223 for dir_path, _, files in os.walk(self._source_directory):
224 for name in files:
225 if name.lower().endswith(self._source_file_extension):
226 basename = os.path.splitext(os.path.basename(name))[0]
227 relative_path = os.path.relpath(os.path.join(dir_path, name))
229 if basename in self._source_file_names:
230 self._io.error("Files '{0}' and '{1}' have the same basename.".
231 format(self._source_file_names[basename], relative_path))
232 self.error_file_names.add(relative_path)
233 else:
234 self._source_file_names[basename] = relative_path
236 # ------------------------------------------------------------------------------------------------------------------
237 def __read_stored_routine_metadata(self) -> None:
238 """
239 Reads the metadata of stored routines from the metadata file.
240 """
241 if os.path.isfile(self._pystratum_metadata_filename):
242 with open(self._pystratum_metadata_filename, 'r') as file:
243 self._pystratum_metadata = json.load(file)
245 # ------------------------------------------------------------------------------------------------------------------
246 @abc.abstractmethod
247 def _get_column_type(self) -> None:
248 """
249 Selects schema, table, column names and the column type from the RDBMS instance and saves them as replace pairs.
250 """
251 raise NotImplementedError()
253 # ------------------------------------------------------------------------------------------------------------------
254 @abc.abstractmethod
255 def create_routine_loader_helper(self,
256 routine_name: str,
257 pystratum_old_metadata: Dict,
258 rdbms_old_metadata: Dict) -> RoutineLoaderHelper:
259 """
260 Creates a Routine Loader Helper object.
262 :param str routine_name: The name of the routine.
263 :param dict pystratum_old_metadata: The old metadata of the stored routine from PyStratum.
264 :param dict rdbms_old_metadata: The old metadata of the stored routine from database instance.
266 :rtype: RoutineLoaderHelper
267 """
268 raise NotImplementedError()
270 # ------------------------------------------------------------------------------------------------------------------
271 def __load_stored_routines(self) -> None:
272 """
273 Loads all stored routines into the RDBMS instance.
274 """
275 self._io.writeln('')
277 for routine_name in sorted(self._source_file_names):
278 if routine_name in self._pystratum_metadata:
279 old_metadata = self._pystratum_metadata[routine_name]
280 else:
281 old_metadata = None
283 if routine_name in self._rdbms_old_metadata:
284 old_routine_info = self._rdbms_old_metadata[routine_name]
285 else:
286 old_routine_info = None
288 routine_loader_helper = self.create_routine_loader_helper(routine_name, old_metadata, old_routine_info)
289 routine_loader_helper.shadow_directory = self.__shadow_directory
291 metadata = routine_loader_helper.load_stored_routine()
293 if not metadata:
294 self.error_file_names.add(self._source_file_names[routine_name])
295 if routine_name in self._pystratum_metadata:
296 del self._pystratum_metadata[routine_name]
297 else:
298 self._pystratum_metadata[routine_name] = metadata
300 # ------------------------------------------------------------------------------------------------------------------
301 @abc.abstractmethod
302 def _get_old_stored_routine_info(self) -> None:
303 """
304 Retrieves information about all stored routines in the current schema.
305 """
306 raise NotImplementedError()
308 # ------------------------------------------------------------------------------------------------------------------
309 def _get_correct_sql_mode(self) -> None:
310 """
311 Gets the SQL mode in the order as preferred by MySQL. This method is specific for MySQL.
312 """
313 pass
315 # ------------------------------------------------------------------------------------------------------------------
316 @abc.abstractmethod
317 def _drop_obsolete_routines(self) -> None:
318 """
319 Drops obsolete stored routines (i.e. stored routines that exits in the current schema but for
320 which we don't have a source file).
321 """
322 raise NotImplementedError()
324 # ------------------------------------------------------------------------------------------------------------------
325 def __remove_obsolete_metadata(self) -> None:
326 """
327 Removes obsolete entries from the metadata of all stored routines.
328 """
329 clean = {}
330 for key, _ in self._source_file_names.items():
331 if key in self._pystratum_metadata:
332 clean[key] = self._pystratum_metadata[key]
334 self._pystratum_metadata = clean
336 # ------------------------------------------------------------------------------------------------------------------
337 def __write_stored_routine_metadata(self) -> None:
338 """
339 Writes the metadata of all stored routines to the metadata file.
340 """
341 with open(self._pystratum_metadata_filename, 'w') as stream:
342 json.dump(self._pystratum_metadata, stream, indent=4, sort_keys=True)
344 # ------------------------------------------------------------------------------------------------------------------
345 def find_source_files_from_list(self, file_names) -> None:
346 """
347 Finds all source files that actually exists from a list of file names.
349 :param list[str] file_names: The list of file names.
350 """
351 for file_name in file_names:
352 if os.path.exists(file_name):
353 routine_name = os.path.splitext(os.path.basename(file_name))[0]
354 if routine_name not in self._source_file_names:
355 self._source_file_names[routine_name] = file_name
356 else:
357 self._io.error("Files '{0}' and '{1}' have the same basename.".
358 format(self._source_file_names[routine_name], file_name))
359 self.error_file_names.add(file_name)
360 else:
361 self._io.error("File not exists: '{0}'".format(file_name))
362 self.error_file_names.add(file_name)
364 # ------------------------------------------------------------------------------------------------------------------
365 def __get_constants(self) -> None:
366 """
367 Gets the constants from the class that acts like a namespace for constants and adds them to the replace pairs.
368 """
369 helper = ConstantClass(self._constants_class_name, self._io)
370 helper.reload()
371 constants = helper.constants()
373 for name, value in constants.items():
374 self._add_replace_pair(name, value, True)
376 self._io.text('Read {0} constants for substitution from <fso>{1}</fso>'.
377 format(len(constants), helper.file_name()))
379# ----------------------------------------------------------------------------------------------------------------------