Coverage for pystratum_common/backend/CommonRoutineLoaderWorker.py: 0%

166 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-13 08:46 +0200

1import abc 

2import configparser 

3import json 

4import os 

5from typing import Dict, List, Optional 

6 

7from pystratum_backend.RoutineLoaderWorker import RoutineLoaderWorker 

8from pystratum_backend.StratumIO import StratumIO 

9 

10from pystratum_common.ConstantClass import ConstantClass 

11from pystratum_common.helper.RoutineLoaderHelper import RoutineLoaderHelper 

12 

13 

14class CommonRoutineLoaderWorker(RoutineLoaderWorker): 

15 """ 

16 Class for loading stored routines into a RDBMS instance from (pseudo) SQL files. 

17 """ 

18 

19 # ------------------------------------------------------------------------------------------------------------------ 

20 def __init__(self, io: StratumIO, config: configparser.ConfigParser): 

21 """ 

22 Object constructor. 

23 

24 :param io: The output decorator. 

25 """ 

26 self.error_file_names = set() 

27 """ 

28 A set with source names that are not loaded into RDBMS instance. 

29 """ 

30 

31 self._pystratum_metadata: Dict = {} 

32 """ 

33 The meta data of all stored routines. 

34 """ 

35 

36 self._pystratum_metadata_filename: Optional[str] = None 

37 """ 

38 The filename of the file with the metadata of all stored routines. 

39 """ 

40 

41 self._rdbms_old_metadata: Dict = {} 

42 """ 

43 Old metadata about all stored routines. 

44 """ 

45 

46 self._replace_pairs: Dict = {} 

47 """ 

48 A map from placeholders to their actual values. 

49 """ 

50 

51 self._source_file_encoding: Optional[str] = None 

52 """ 

53 The character set of the source files. 

54 """ 

55 

56 self._source_directory: Optional[str] = None 

57 """ 

58 Path where source files can be found. 

59 """ 

60 

61 self._source_file_extension: Optional[str] = None 

62 """ 

63 The extension of the source files. 

64 """ 

65 

66 self._source_file_names: Dict = {} 

67 """ 

68 All found source files. 

69 """ 

70 

71 self._constants_class_name: str = '' 

72 """ 

73 The name of the class that acts like a namespace for constants. 

74 """ 

75 

76 self.__shadow_directory: Optional[str] = None 

77 """ 

78 The name of the directory were copies with pure SQL of the stored routine sources must be stored. 

79 """ 

80 

81 self._io: StratumIO = io 

82 """ 

83 The output decorator. 

84 """ 

85 

86 self._config = config 

87 """ 

88 The configuration object. 

89 """ 

90 

91 # ------------------------------------------------------------------------------------------------------------------ 

92 def execute(self, file_names: Optional[List[str]] = None) -> int: 

93 """ 

94 Loads stored routines into the current schema. 

95 

96 :param list[str] file_names: The sources that must be loaded. If empty all sources (if required) will loaded. 

97 

98 :rtype: int The status of exit. 

99 """ 

100 self._io.title('Loader') 

101 

102 self._read_configuration_file() 

103 

104 if file_names: 

105 self.__load_list(file_names) 

106 else: 

107 self.__load_all() 

108 

109 if self.error_file_names: 

110 self.__log_overview_errors() 

111 

112 self._io.write_line('') 

113 

114 return 1 if self.error_file_names else 0 

115 

116 # ------------------------------------------------------------------------------------------------------------------ 

117 def __log_overview_errors(self) -> None: 

118 """ 

119 Show info about sources files of stored routines that were not loaded successfully. 

120 """ 

121 if self.error_file_names: 

122 self._io.warning('Routines in the files below are not loaded:') 

123 self._io.listing(sorted(self.error_file_names)) 

124 

125 # ------------------------------------------------------------------------------------------------------------------ 

126 @abc.abstractmethod 

127 def connect(self) -> None: 

128 """ 

129 Connects to the RDBMS instance. 

130 """ 

131 raise NotImplementedError() 

132 

133 # ------------------------------------------------------------------------------------------------------------------ 

134 @abc.abstractmethod 

135 def disconnect(self) -> None: 

136 """ 

137 Disconnects from the RDBMS instance. 

138 """ 

139 raise NotImplementedError() 

140 

141 # ------------------------------------------------------------------------------------------------------------------ 

142 def _add_replace_pair(self, name: str, value: str, quote: bool): 

143 """ 

144 Adds a replacement to the map of replacement pairs. 

145 

146 :param name: The name of the replacement pair. 

147 :param value: The value of the replacement pair. 

148 """ 

149 key = '@' + name + '@' 

150 key = key.lower() 

151 

152 class_name = value.__class__.__name__ 

153 

154 if class_name in ['int', 'float']: 

155 value = str(value) 

156 elif class_name in ['bool']: 

157 value = '1' if value else '0' 

158 elif class_name in ['str']: 

159 if quote: 

160 value = "'" + value + "'" 

161 else: 

162 self._io.log_verbose("Ignoring constant {} which is an instance of {}".format(name, class_name)) 

163 

164 self._replace_pairs[key] = value 

165 

166 # ------------------------------------------------------------------------------------------------------------------ 

167 def __load_list(self, file_names: Optional[List[str]]) -> None: 

168 """ 

169 Loads all stored routines in a list into the RDBMS instance. 

170 

171 :param file_names: The list of files to be loaded. 

172 """ 

173 self.connect() 

174 self.find_source_files_from_list(file_names) 

175 self._get_column_type() 

176 self.__read_stored_routine_metadata() 

177 self.__get_constants() 

178 self._get_old_stored_routine_info() 

179 self._get_correct_sql_mode() 

180 self.__load_stored_routines() 

181 self.__write_stored_routine_metadata() 

182 self.disconnect() 

183 

184 # ------------------------------------------------------------------------------------------------------------------ 

185 def __load_all(self) -> None: 

186 """ 

187 Loads all stored routines into the RDBMS instance. 

188 """ 

189 self.connect() 

190 self.__find_source_files() 

191 self._get_column_type() 

192 self.__read_stored_routine_metadata() 

193 self.__get_constants() 

194 self._get_old_stored_routine_info() 

195 self._get_correct_sql_mode() 

196 self.__load_stored_routines() 

197 self._drop_obsolete_routines() 

198 self.__remove_obsolete_metadata() 

199 self.__write_stored_routine_metadata() 

200 self.disconnect() 

201 

202 # ------------------------------------------------------------------------------------------------------------------ 

203 def _read_configuration_file(self) -> None: 

204 """ 

205 Reads parameters from the configuration file. 

206 """ 

207 self._source_directory = self._config.get('loader', 'source_directory') 

208 self._source_file_extension = self._config.get('loader', 'extension') 

209 self._source_file_encoding = self._config.get('loader', 'encoding') 

210 self.__shadow_directory = self._config.get('loader', 'shadow_directory', fallback=None) 

211 self._pystratum_metadata_filename = self._config.get('wrapper', 'metadata') 

212 self._constants_class_name = self._config.get('constants', 'class') 

213 

214 # ------------------------------------------------------------------------------------------------------------------ 

215 def __find_source_files(self) -> None: 

216 """ 

217 Searches recursively for all source files in a directory. 

218 """ 

219 for dir_path, _, files in os.walk(self._source_directory): 

220 for name in files: 

221 if name.lower().endswith(self._source_file_extension): 

222 basename = os.path.splitext(os.path.basename(name))[0] 

223 relative_path = os.path.relpath(os.path.join(dir_path, name)) 

224 

225 if basename in self._source_file_names: 

226 self._io.error("Files '{0}' and '{1}' have the same basename.".format( 

227 self._source_file_names[basename], relative_path)) 

228 self.error_file_names.add(relative_path) 

229 else: 

230 self._source_file_names[basename] = relative_path 

231 

232 # ------------------------------------------------------------------------------------------------------------------ 

233 def __read_stored_routine_metadata(self) -> None: 

234 """ 

235 Reads the metadata of stored routines from the metadata file. 

236 """ 

237 if os.path.isfile(self._pystratum_metadata_filename): 

238 with open(self._pystratum_metadata_filename, 'r') as file: 

239 self._pystratum_metadata = json.load(file) 

240 

241 # ------------------------------------------------------------------------------------------------------------------ 

242 @abc.abstractmethod 

243 def _get_column_type(self) -> None: 

244 """ 

245 Selects schema, table, column names and the column type from the RDBMS instance and saves them as replace pairs. 

246 """ 

247 raise NotImplementedError() 

248 

249 # ------------------------------------------------------------------------------------------------------------------ 

250 @abc.abstractmethod 

251 def create_routine_loader_helper(self, 

252 routine_name: str, 

253 pystratum_old_metadata: Dict, 

254 rdbms_old_metadata: Dict) -> RoutineLoaderHelper: 

255 """ 

256 Creates a Routine Loader Helper object. 

257 

258 :param routine_name: The name of the routine. 

259 :param pystratum_old_metadata: The old metadata of the stored routine from PyStratum. 

260 :param rdbms_old_metadata: The old metadata of the stored routine from database instance. 

261 """ 

262 raise NotImplementedError() 

263 

264 # ------------------------------------------------------------------------------------------------------------------ 

265 def __load_stored_routines(self) -> None: 

266 """ 

267 Loads all stored routines into the RDBMS instance. 

268 """ 

269 self._io.write_line('') 

270 

271 for routine_name in sorted(self._source_file_names): 

272 if routine_name in self._pystratum_metadata: 

273 old_metadata = self._pystratum_metadata[routine_name] 

274 else: 

275 old_metadata = None 

276 

277 if routine_name in self._rdbms_old_metadata: 

278 old_routine_info = self._rdbms_old_metadata[routine_name] 

279 else: 

280 old_routine_info = None 

281 

282 routine_loader_helper = self.create_routine_loader_helper(routine_name, old_metadata, old_routine_info) 

283 routine_loader_helper.shadow_directory = self.__shadow_directory 

284 

285 metadata = routine_loader_helper.load_stored_routine() 

286 

287 if not metadata: 

288 self.error_file_names.add(self._source_file_names[routine_name]) 

289 if routine_name in self._pystratum_metadata: 

290 del self._pystratum_metadata[routine_name] 

291 else: 

292 self._pystratum_metadata[routine_name] = metadata 

293 

294 # ------------------------------------------------------------------------------------------------------------------ 

295 @abc.abstractmethod 

296 def _get_old_stored_routine_info(self) -> None: 

297 """ 

298 Retrieves information about all stored routines in the current schema. 

299 """ 

300 raise NotImplementedError() 

301 

302 # ------------------------------------------------------------------------------------------------------------------ 

303 def _get_correct_sql_mode(self) -> None: 

304 """ 

305 Gets the SQL mode in the order as preferred by MySQL. This method is specific for MySQL. 

306 """ 

307 pass 

308 

309 # ------------------------------------------------------------------------------------------------------------------ 

310 @abc.abstractmethod 

311 def _drop_obsolete_routines(self) -> None: 

312 """ 

313 Drops obsolete stored routines (i.e. stored routines that exits in the current schema but for 

314 which we don't have a source file). 

315 """ 

316 raise NotImplementedError() 

317 

318 # ------------------------------------------------------------------------------------------------------------------ 

319 def __remove_obsolete_metadata(self) -> None: 

320 """ 

321 Removes obsolete entries from the metadata of all stored routines. 

322 """ 

323 clean = {} 

324 for key, _ in self._source_file_names.items(): 

325 if key in self._pystratum_metadata: 

326 clean[key] = self._pystratum_metadata[key] 

327 

328 self._pystratum_metadata = clean 

329 

330 # ------------------------------------------------------------------------------------------------------------------ 

331 def __write_stored_routine_metadata(self) -> None: 

332 """ 

333 Writes the metadata of all stored routines to the metadata file. 

334 """ 

335 with open(self._pystratum_metadata_filename, 'w') as stream: 

336 json.dump(self._pystratum_metadata, stream, indent=4, sort_keys=True) 

337 

338 # ------------------------------------------------------------------------------------------------------------------ 

339 def find_source_files_from_list(self, file_names) -> None: 

340 """ 

341 Finds all source files that actually exists from a list of file names. 

342 

343 :param file_names: The list of file names. 

344 """ 

345 for file_name in file_names: 

346 if os.path.exists(file_name): 

347 routine_name = os.path.splitext(os.path.basename(file_name))[0] 

348 if routine_name not in self._source_file_names: 

349 self._source_file_names[routine_name] = file_name 

350 else: 

351 self._io.error("Files '{0}' and '{1}' have the same basename.".format( 

352 self._source_file_names[routine_name], file_name)) 

353 self.error_file_names.add(file_name) 

354 else: 

355 self._io.error("File not exists: '{0}'".format(file_name)) 

356 self.error_file_names.add(file_name) 

357 

358 # ------------------------------------------------------------------------------------------------------------------ 

359 def __get_constants(self) -> None: 

360 """ 

361 Gets the constants from the class that acts like a namespace for constants and adds them to the replacement 

362 pairs. 

363 """ 

364 helper = ConstantClass(self._constants_class_name, self._io) 

365 helper.reload() 

366 constants = helper.constants() 

367 

368 for name, value in constants.items(): 

369 self._add_replace_pair(name, value, True) 

370 

371 self._io.text('Read {0} constants for substitution from <fso>{1}</fso>'.format(len(constants), 

372 helper.file_name())) 

373 

374# ----------------------------------------------------------------------------------------------------------------------