Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import abc 

2import configparser 

3import json 

4import os 

5from typing import Dict, List, Optional 

6 

7from pystratum_backend.RoutineLoaderWorker import RoutineLoaderWorker 

8from pystratum_backend.StratumStyle import StratumStyle 

9from pystratum_common.ConstantClass import ConstantClass 

10from pystratum_common.helper.RoutineLoaderHelper import RoutineLoaderHelper 

11 

12 

13class CommonRoutineLoaderWorker(RoutineLoaderWorker): 

14 """ 

15 Class for loading stored routines into a RDBMS instance from (pseudo) SQL files. 

16 """ 

17 

18 # ------------------------------------------------------------------------------------------------------------------ 

19 def __init__(self, io: StratumStyle, config: configparser.ConfigParser): 

20 """ 

21 Object constructor. 

22 

23 :param pystratum.style.PyStratumStyle.PyStratumStyle io: The output decorator. 

24 """ 

25 self.error_file_names = set() 

26 """ 

27 A set with source names that are not loaded into RDBMS instance. 

28 

29 :type: set 

30 """ 

31 

32 self._pystratum_metadata: Dict = {} 

33 """ 

34 The meta data of all stored routines. 

35 """ 

36 

37 self._pystratum_metadata_filename: Optional[str] = None 

38 """ 

39 The filename of the file with the metadata of all stored routines. 

40 """ 

41 

42 self._rdbms_old_metadata: Dict = {} 

43 """ 

44 Old metadata about all stored routines. 

45 """ 

46 

47 self._replace_pairs: Dict = {} 

48 """ 

49 A map from placeholders to their actual values. 

50 """ 

51 

52 self._source_file_encoding: Optional[str] = None 

53 """ 

54 The character set of the source files. 

55 """ 

56 

57 self._source_directory: Optional[str] = None 

58 """ 

59 Path where source files can be found. 

60 """ 

61 

62 self._source_file_extension: Optional[str] = None 

63 """ 

64 The extension of the source files. 

65 """ 

66 

67 self._source_file_names: Dict = {} 

68 """ 

69 All found source files. 

70 """ 

71 

72 self._constants_class_name: str = '' 

73 """ 

74 The name of the class that acts like a namespace for constants. 

75 """ 

76 

77 self.__shadow_directory: Optional[str] = None 

78 """ 

79 The name of the directory were copies with pure SQL of the stored routine sources must be stored. 

80 """ 

81 

82 self._io: StratumStyle = io 

83 """ 

84 The output decorator. 

85 """ 

86 

87 self._config = config 

88 """ 

89 The configuration object. 

90 

91 :type: ConfigParser  

92 """ 

93 

94 # ------------------------------------------------------------------------------------------------------------------ 

95 def execute(self, file_names: Optional[List[str]] = None) -> int: 

96 """ 

97 Loads stored routines into the current schema. 

98 

99 :param list[str] file_names: The sources that must be loaded. If empty all sources (if required) will loaded. 

100 

101 :rtype: int The status of exit. 

102 """ 

103 self._io.title('Loader') 

104 

105 self._read_configuration_file() 

106 

107 if file_names: 

108 self.__load_list(file_names) 

109 else: 

110 self.__load_all() 

111 

112 if self.error_file_names: 

113 self.__log_overview_errors() 

114 

115 self._io.writeln('') 

116 

117 return 1 if self.error_file_names else 0 

118 

119 # ------------------------------------------------------------------------------------------------------------------ 

120 def __log_overview_errors(self) -> None: 

121 """ 

122 Show info about sources files of stored routines that were not loaded successfully. 

123 """ 

124 if self.error_file_names: 

125 self._io.warning('Routines in the files below are not loaded:') 

126 self._io.listing(sorted(self.error_file_names)) 

127 

128 # ------------------------------------------------------------------------------------------------------------------ 

129 @abc.abstractmethod 

130 def connect(self) -> None: 

131 """ 

132 Connects to the RDBMS instance. 

133 """ 

134 raise NotImplementedError() 

135 

136 # ------------------------------------------------------------------------------------------------------------------ 

137 @abc.abstractmethod 

138 def disconnect(self) -> None: 

139 """ 

140 Disconnects from the RDBMS instance. 

141 """ 

142 raise NotImplementedError() 

143 

144 # ------------------------------------------------------------------------------------------------------------------ 

145 def _add_replace_pair(self, name: str, value: str, quote: bool): 

146 """ 

147 Adds a replace part to the map of replace pairs. 

148 

149 :param name: The name of the replace pair. 

150 :param value: The value of value of the replace pair. 

151 """ 

152 key = '@' + name + '@' 

153 key = key.lower() 

154 

155 class_name = value.__class__.__name__ 

156 

157 if class_name in ['int', 'float']: 

158 value = str(value) 

159 elif class_name in ['bool']: 

160 value = '1' if value else '0' 

161 elif class_name in ['str']: 

162 if quote: 

163 value = "'" + value + "'" 

164 else: 

165 self._io.log_verbose("Ignoring constant {} which is an instance of {}".format(name, class_name)) 

166 

167 self._replace_pairs[key] = value 

168 

169 # ------------------------------------------------------------------------------------------------------------------ 

170 def __load_list(self, file_names: Optional[List[str]]) -> None: 

171 """ 

172 Loads all stored routines in a list into the RDBMS instance. 

173 :param list[str] file_names: The list of files to be loaded. 

174 """ 

175 self.connect() 

176 self.find_source_files_from_list(file_names) 

177 self._get_column_type() 

178 self.__read_stored_routine_metadata() 

179 self.__get_constants() 

180 self._get_old_stored_routine_info() 

181 self._get_correct_sql_mode() 

182 self.__load_stored_routines() 

183 self.__write_stored_routine_metadata() 

184 self.disconnect() 

185 

186 # ------------------------------------------------------------------------------------------------------------------ 

187 def __load_all(self) -> None: 

188 """ 

189 Loads all stored routines into the RDBMS instance. 

190 """ 

191 self.connect() 

192 self.__find_source_files() 

193 self._get_column_type() 

194 self.__read_stored_routine_metadata() 

195 self.__get_constants() 

196 self._get_old_stored_routine_info() 

197 self._get_correct_sql_mode() 

198 self.__load_stored_routines() 

199 self._drop_obsolete_routines() 

200 self.__remove_obsolete_metadata() 

201 self.__write_stored_routine_metadata() 

202 self.disconnect() 

203 

204 # ------------------------------------------------------------------------------------------------------------------ 

205 def _read_configuration_file(self) -> None: 

206 """ 

207 Reads parameters from the configuration file. 

208 """ 

209 self._source_directory = self._config.get('loader', 'source_directory') 

210 self._source_file_extension = self._config.get('loader', 'extension') 

211 self._source_file_encoding = self._config.get('loader', 'encoding') 

212 self.__shadow_directory = self._config.get('loader', 'shadow_directory', fallback=None) 

213 

214 self._pystratum_metadata_filename = self._config.get('wrapper', 'metadata') 

215 

216 self._constants_class_name = self._config.get('constants', 'class') 

217 

218 # ------------------------------------------------------------------------------------------------------------------ 

219 def __find_source_files(self) -> None: 

220 """ 

221 Searches recursively for all source files in a directory. 

222 """ 

223 for dir_path, _, files in os.walk(self._source_directory): 

224 for name in files: 

225 if name.lower().endswith(self._source_file_extension): 

226 basename = os.path.splitext(os.path.basename(name))[0] 

227 relative_path = os.path.relpath(os.path.join(dir_path, name)) 

228 

229 if basename in self._source_file_names: 

230 self._io.error("Files '{0}' and '{1}' have the same basename.". 

231 format(self._source_file_names[basename], relative_path)) 

232 self.error_file_names.add(relative_path) 

233 else: 

234 self._source_file_names[basename] = relative_path 

235 

236 # ------------------------------------------------------------------------------------------------------------------ 

237 def __read_stored_routine_metadata(self) -> None: 

238 """ 

239 Reads the metadata of stored routines from the metadata file. 

240 """ 

241 if os.path.isfile(self._pystratum_metadata_filename): 

242 with open(self._pystratum_metadata_filename, 'r') as file: 

243 self._pystratum_metadata = json.load(file) 

244 

245 # ------------------------------------------------------------------------------------------------------------------ 

246 @abc.abstractmethod 

247 def _get_column_type(self) -> None: 

248 """ 

249 Selects schema, table, column names and the column type from the RDBMS instance and saves them as replace pairs. 

250 """ 

251 raise NotImplementedError() 

252 

253 # ------------------------------------------------------------------------------------------------------------------ 

254 @abc.abstractmethod 

255 def create_routine_loader_helper(self, 

256 routine_name: str, 

257 pystratum_old_metadata: Dict, 

258 rdbms_old_metadata: Dict) -> RoutineLoaderHelper: 

259 """ 

260 Creates a Routine Loader Helper object. 

261 

262 :param str routine_name: The name of the routine. 

263 :param dict pystratum_old_metadata: The old metadata of the stored routine from PyStratum. 

264 :param dict rdbms_old_metadata: The old metadata of the stored routine from database instance. 

265 

266 :rtype: RoutineLoaderHelper 

267 """ 

268 raise NotImplementedError() 

269 

270 # ------------------------------------------------------------------------------------------------------------------ 

271 def __load_stored_routines(self) -> None: 

272 """ 

273 Loads all stored routines into the RDBMS instance. 

274 """ 

275 self._io.writeln('') 

276 

277 for routine_name in sorted(self._source_file_names): 

278 if routine_name in self._pystratum_metadata: 

279 old_metadata = self._pystratum_metadata[routine_name] 

280 else: 

281 old_metadata = None 

282 

283 if routine_name in self._rdbms_old_metadata: 

284 old_routine_info = self._rdbms_old_metadata[routine_name] 

285 else: 

286 old_routine_info = None 

287 

288 routine_loader_helper = self.create_routine_loader_helper(routine_name, old_metadata, old_routine_info) 

289 routine_loader_helper.shadow_directory = self.__shadow_directory 

290 

291 metadata = routine_loader_helper.load_stored_routine() 

292 

293 if not metadata: 

294 self.error_file_names.add(self._source_file_names[routine_name]) 

295 if routine_name in self._pystratum_metadata: 

296 del self._pystratum_metadata[routine_name] 

297 else: 

298 self._pystratum_metadata[routine_name] = metadata 

299 

300 # ------------------------------------------------------------------------------------------------------------------ 

301 @abc.abstractmethod 

302 def _get_old_stored_routine_info(self) -> None: 

303 """ 

304 Retrieves information about all stored routines in the current schema. 

305 """ 

306 raise NotImplementedError() 

307 

308 # ------------------------------------------------------------------------------------------------------------------ 

309 def _get_correct_sql_mode(self) -> None: 

310 """ 

311 Gets the SQL mode in the order as preferred by MySQL. This method is specific for MySQL. 

312 """ 

313 pass 

314 

315 # ------------------------------------------------------------------------------------------------------------------ 

316 @abc.abstractmethod 

317 def _drop_obsolete_routines(self) -> None: 

318 """ 

319 Drops obsolete stored routines (i.e. stored routines that exits in the current schema but for 

320 which we don't have a source file). 

321 """ 

322 raise NotImplementedError() 

323 

324 # ------------------------------------------------------------------------------------------------------------------ 

325 def __remove_obsolete_metadata(self) -> None: 

326 """ 

327 Removes obsolete entries from the metadata of all stored routines. 

328 """ 

329 clean = {} 

330 for key, _ in self._source_file_names.items(): 

331 if key in self._pystratum_metadata: 

332 clean[key] = self._pystratum_metadata[key] 

333 

334 self._pystratum_metadata = clean 

335 

336 # ------------------------------------------------------------------------------------------------------------------ 

337 def __write_stored_routine_metadata(self) -> None: 

338 """ 

339 Writes the metadata of all stored routines to the metadata file. 

340 """ 

341 with open(self._pystratum_metadata_filename, 'w') as stream: 

342 json.dump(self._pystratum_metadata, stream, indent=4, sort_keys=True) 

343 

344 # ------------------------------------------------------------------------------------------------------------------ 

345 def find_source_files_from_list(self, file_names) -> None: 

346 """ 

347 Finds all source files that actually exists from a list of file names. 

348 

349 :param list[str] file_names: The list of file names. 

350 """ 

351 for file_name in file_names: 

352 if os.path.exists(file_name): 

353 routine_name = os.path.splitext(os.path.basename(file_name))[0] 

354 if routine_name not in self._source_file_names: 

355 self._source_file_names[routine_name] = file_name 

356 else: 

357 self._io.error("Files '{0}' and '{1}' have the same basename.". 

358 format(self._source_file_names[routine_name], file_name)) 

359 self.error_file_names.add(file_name) 

360 else: 

361 self._io.error("File not exists: '{0}'".format(file_name)) 

362 self.error_file_names.add(file_name) 

363 

364 # ------------------------------------------------------------------------------------------------------------------ 

365 def __get_constants(self) -> None: 

366 """ 

367 Gets the constants from the class that acts like a namespace for constants and adds them to the replace pairs. 

368 """ 

369 helper = ConstantClass(self._constants_class_name, self._io) 

370 helper.reload() 

371 constants = helper.constants() 

372 

373 for name, value in constants.items(): 

374 self._add_replace_pair(name, value, True) 

375 

376 self._io.text('Read {0} constants for substitution from <fso>{1}</fso>'. 

377 format(len(constants), helper.file_name())) 

378 

379# ----------------------------------------------------------------------------------------------------------------------