Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/ilm.py: 25%

143 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-08-30 20:56 -0600

1"""ILM Defining Class""" 

2 

3import typing as t 

4import logging 

5from os import getenv 

6from dotmap import DotMap 

7from elasticsearch8.exceptions import BadRequestError 

8from es_wait import IlmPhase, IlmStep 

9from es_wait.exceptions import IlmWaitError 

10from es_testbed.defaults import ( 

11 PAUSE_ENVVAR, 

12 PAUSE_DEFAULT, 

13 TIMEOUT_DEFAULT, 

14 TIMEOUT_ENVVAR, 

15) 

16from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig 

17from es_testbed.helpers.es_api import get_ilm_phases, ilm_explain, ilm_move, resolver 

18from es_testbed.helpers.utils import prettystr 

19 

20if t.TYPE_CHECKING: 

21 from elasticsearch8 import Elasticsearch 

22 

23PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

24TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT)) 

25 

26logger = logging.getLogger('es_testbed.IlmTracker') 

27 

28# ## Example ILM explain output 

29# { 

30# 'action': 'complete', 

31# 'action_time_millis': 0, 

32# 'age': '5.65m', 

33# 'index': 'INDEX_NAME', 

34# 'index_creation_date_millis': 0, 

35# 'lifecycle_date_millis': 0, 

36# 'managed': True, 

37# 'phase': 'hot', 

38# 'phase_execution': { 

39# 'modified_date_in_millis': 0, 

40# 'phase_definition': { 

41# 'actions': { 

42# 'rollover': { 

43# 'max_age': 'MAX_AGE', 

44# 'max_primary_shard_docs': 1000, 

45# 'max_primary_shard_size': 'MAX_SIZE', 

46# 'min_docs': 1 

47# } 

48# }, 

49# 'min_age': '0ms' 

50# }, 

51# 'policy': 'POLICY_NAME', 

52# 'version': 1 

53# }, 

54# 'phase_time_millis': 0, 

55# 'policy': 'POLICY_NAME', 

56# 'step': 'complete', 

57# 'step_time_millis': 0, 

58# 'time_since_index_creation': '5.65m' 

59# } 

60 

61 

62class IlmTracker: 

63 """ILM Phase Tracking Class""" 

64 

65 def __init__(self, client: 'Elasticsearch', name: str): 

66 self.client = client 

67 self.name = self.resolve(name) # A single index name 

68 self._explain = DotMap(self.get_explain_data()) 

69 self._phases = get_ilm_phases(self.client, self._explain.policy) 

70 

71 @property 

72 def current_step(self) -> t.Dict: 

73 """Return the current ILM step information""" 

74 self.update() 

75 return { 

76 'phase': self._explain.phase, 

77 'action': self._explain.action, 

78 'name': self._explain.step, 

79 } 

80 

81 @property 

82 def explain(self) -> DotMap: 

83 """Return the current stored value of ILM Explain""" 

84 return self._explain 

85 

86 @property 

87 def next_phase(self) -> str: 

88 """Return the next phase in the index's ILM journey""" 

89 retval = None 

90 if self._explain.phase == 'delete': 

91 logger.warning('Already on "delete" phase. No more phases to advance') 

92 else: 

93 curr = self.pnum(self._explain.phase) # A numeric representation 

94 # A list of any remaining phases in the policy with a higher number than 

95 # the current 

96 remaining = [ 

97 self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr 

98 ] 

99 if remaining: # If any: 

100 retval = self.pname(remaining[0]) 

101 # Get the phase name from the number stored in the first element 

102 return retval 

103 

104 @property 

105 def policy_phases(self) -> t.Sequence[str]: 

106 """Return a list of phases in the ILM policy""" 

107 return list(self._phases.keys()) 

108 

109 def _log_phase(self, phase: str) -> None: 

110 logger.debug('ILM Explain Index: %s', self._explain.index) 

111 logger.info('Index "%s" now on phase "%s"', self.name, phase) 

112 

113 def _phase_wait( 

114 self, phase: str, pause: float = PAUSE_VALUE, timeout: float = TIMEOUT_VALUE 

115 ) -> None: 

116 """Wait until the new phase shows up in ILM Explain""" 

117 kw = {'name': self.name, 'phase': phase, 'pause': pause, 'timeout': timeout} 

118 phasechk = IlmPhase(self.client, **kw) 

119 phasechk.wait() 

120 

121 def _ssphz(self, phase: str) -> bool: 

122 return bool(self.pnum(phase) > self.pnum('warm')) 

123 

124 def advance( 

125 self, 

126 phase: t.Union[str, None] = None, 

127 action: t.Union[str, None] = None, 

128 name: t.Union[str, None] = None, 

129 ) -> None: 

130 """Advance index to next ILM phase""" 

131 if self._explain.phase == 'delete': 

132 logger.warning('Already on "delete" phase. No more phases to advance') 

133 else: 

134 logger.debug('current_step: %s', prettystr(self.current_step)) 

135 next_step = self.next_step(phase, action=action, name=name) 

136 logger.debug('next_step: %s', prettystr(next_step)) 

137 if self._explain.phase == 'new' and phase == 'hot': 

138 # It won't be for very long. 

139 self._phase_wait('hot') 

140 

141 # Regardless of the remaining phases, the current phase steps must be 

142 # complete before proceeding with ilm_move 

143 self.update() 

144 self.wait4complete() 

145 self.update() 

146 

147 # We could have arrived with it hot, but incomplete 

148 if phase == 'hot': 

149 self._log_phase(phase) 

150 # we've advanced to our target phase, and all steps are completed 

151 

152 # Remaining phases could be warm through frozen 

153 elif self._explain.phase != phase: 

154 

155 # We will only wait for steps to complete for the hot and warm tiers 

156 wait4steps = False if self._ssphz(phase) else False 

157 

158 ilm_move(self.client, self.name, self.current_step, next_step) 

159 self._phase_wait(phase) 

160 # If cold or frozen, we can return now. We let the calling function 

161 # worry about the weird name changing behavior of searchable mounts 

162 

163 if wait4steps: 

164 self.update() 

165 logger.debug( 

166 'Waiting for "%s" phase steps to complete...', 

167 phase, 

168 ) 

169 self.wait4complete() 

170 self.update() 

171 self._log_phase(phase) 

172 else: 

173 logger.error('next_step is a None value') 

174 logger.error('current_step: %s', prettystr(self.current_step)) 

175 

176 def get_explain_data(self) -> t.Dict: 

177 """Get the ILM explain data and return it""" 

178 try: 

179 return ilm_explain(self.client, self.name) 

180 except NameChanged as err: 

181 logger.debug('Passing along upstream exception...') 

182 raise NameChanged from err 

183 except ResultNotExpected as err: 

184 msg = f'Unable to get ilm_explain results. Error: {prettystr(err)}' 

185 logger.critical(msg) 

186 raise ResultNotExpected(msg) from err 

187 

188 def next_step( 

189 self, 

190 phase: t.Union[str, None] = None, 

191 action: t.Union[str, None] = None, 

192 name: t.Union[str, None] = None, 

193 ) -> t.Dict: 

194 """Determine the next ILM step based on the current phase, action, and name""" 

195 err1 = bool((action is not None) and (name is None)) 

196 err2 = bool((action is None) and (name is not None)) 

197 if err1 or err2: 

198 msg = 'If either action or name is specified, both must be' 

199 logger.critical(msg) 

200 raise TestbedMisconfig(msg) 

201 if not phase: 

202 phase = self.next_phase 

203 retval = {'phase': phase} 

204 if action: 

205 retval['action'] = action 

206 retval['name'] = name 

207 return retval 

208 

209 def pnum(self, phase: str) -> int: 

210 """Map a phase name to a phase number""" 

211 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5} 

212 return _[phase] 

213 

214 def pname(self, num: int) -> str: 

215 """Map a phase number to a phase name""" 

216 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'} 

217 return _[num] 

218 

219 def resolve(self, name: str) -> str: 

220 """Resolve that we have an index and NOT an alias or a datastream""" 

221 res = resolver(self.client, name) 

222 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 

223 msg = f'{name} is not an index: {res}' 

224 logger.critical(msg) 

225 raise ResultNotExpected(msg) 

226 if len(res['indices']) > 1: 

227 msg = f'{name} resolved to multiple indices: {prettystr(res["indices"])}' 

228 logger.critical(msg) 

229 raise ResultNotExpected(msg) 

230 return res['indices'][0]['name'] 

231 

232 def update(self) -> None: 

233 """Update self._explain with the latest from :py:meth:`get_explain_data`""" 

234 try: 

235 self._explain = DotMap(self.get_explain_data()) 

236 except NameChanged as err: 

237 logger.debug('Passing along upstream exception...') 

238 raise NameChanged from err 

239 

240 def wait4complete(self) -> None: 

241 """Subroutine for waiting for an ILM step to complete""" 

242 step_action = bool(self._explain.action == 'complete') 

243 step_name = bool(self._explain.name == 'complete') 

244 if bool(step_action and step_name): 

245 logger.debug( 

246 '%s: Current step complete: %s', self.name, prettystr(self.current_step) 

247 ) 

248 return 

249 logger.debug( 

250 '%s: Current step not complete. %s', self.name, prettystr(self.current_step) 

251 ) 

252 kw = {'name': self.name, 'pause': PAUSE_VALUE, 'timeout': TIMEOUT_VALUE} 

253 step = IlmStep(self.client, **kw) 

254 try: 

255 step.wait() 

256 logger.debug('ILM Step successful. The wait is over') 

257 except KeyError as exc: 

258 logger.error('KeyError: The index name has changed: %s', prettystr(exc)) 

259 raise exc 

260 except BadRequestError as exc: 

261 logger.error('Index not found') 

262 raise exc 

263 except IlmWaitError as exc: 

264 logger.error('Other IlmWait error encountered: %s', prettystr(exc)) 

265 raise exc