Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 83%
114 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-08-30 21:00 -0600
« prev ^ index » next coverage.py v7.4.4, created at 2024-08-30 21:00 -0600
1"""Index Entity Class"""
3import typing as t
4import logging
5from os import getenv
6from elasticsearch8.exceptions import BadRequestError
7from es_wait import Exists, IlmPhase, IlmStep
8from es_wait.exceptions import IlmWaitError
9from es_testbed.defaults import (
10 PAUSE_DEFAULT,
11 PAUSE_ENVVAR,
12 TIMEOUT_DEFAULT,
13 TIMEOUT_ENVVAR,
14)
15from es_testbed.entities.entity import Entity
16from es_testbed.helpers.es_api import snapshot_name
17from es_testbed.helpers.utils import mounted_name, prettystr
18from es_testbed.ilm import IlmTracker
20if t.TYPE_CHECKING:
21 from elasticsearch8 import Elasticsearch
23PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
24TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT))
26logger = logging.getLogger(__name__)
29class Index(Entity):
30 """Index Entity Class"""
32 def __init__(
33 self,
34 client: 'Elasticsearch',
35 name: t.Union[str, None] = None,
36 snapmgr=None,
37 policy_name: str = None,
38 ):
39 super().__init__(client=client, name=name)
40 self.policy_name = policy_name
41 self.ilm_tracker = None
42 self.snapmgr = snapmgr
44 @property
45 def _get_target(self) -> str:
46 target = None
47 phases = self.ilm_tracker.policy_phases
48 curr = self.ilm_tracker.explain.phase
49 if not bool(('cold' in phases) or ('frozen' in phases)):
50 logger.info('ILM Policy for "%s" has no cold/frozen phases', self.name)
51 target = curr # Keep the same
52 if bool(('cold' in phases) and ('frozen' in phases)):
53 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'):
54 target = 'cold'
55 elif curr == 'cold':
56 target = 'frozen'
57 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'):
58 target = curr
59 elif bool(('cold' in phases) and ('frozen' not in phases)):
60 target = 'cold'
61 elif bool(('cold' not in phases) and ('frozen' in phases)):
62 target = 'frozen'
63 return target
65 @property
66 def phase_tuple(self) -> t.Tuple[str, str]:
67 """Return the current phase and the target phase as a Tuple"""
68 return self.ilm_tracker.explain.phase, self._get_target
70 def _add_snap_step(self) -> None:
71 logger.debug('Getting snapshot name for tracking...')
72 snapname = snapshot_name(self.client, self.name)
73 logger.debug('Snapshot %s backs %s', snapname, self.name)
74 self.snapmgr.add_existing(snapname)
76 def _ilm_step(self) -> None:
77 """Subroutine for waiting for an ILM step to complete"""
78 step = {
79 'phase': self.ilm_tracker.explain.phase,
80 'action': self.ilm_tracker.explain.action,
81 'name': self.ilm_tracker.explain.step,
82 }
83 logger.debug('%s: Current Step: %s', self.name, step)
84 step = IlmStep(
85 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name
86 )
87 try:
88 step.wait()
89 logger.debug('ILM Step successful. The wait is over')
90 except KeyError as exc:
91 logger.error('KeyError: The index name has changed: "%s"', exc)
92 raise exc
93 except BadRequestError as exc:
94 logger.error('Index not found')
95 raise exc
96 except IlmWaitError as exc:
97 logger.error('Other IlmWait error encountered: "%s"', exc)
98 raise exc
100 def _mounted_step(self, target: str) -> str:
101 try:
102 self.ilm_tracker.advance(phase=target)
103 except BadRequestError as err:
104 logger.critical('err: %s', prettystr(err))
105 raise BadRequestError from err
106 # At this point, it's "in" a searchable tier, but the index name hasn't
107 # changed yet
108 newidx = mounted_name(self.name, target)
109 logger.debug('Waiting for ILM phase change to complete. New index: %s', newidx)
110 kwargs = {
111 'name': newidx,
112 'kind': 'index',
113 'pause': PAUSE_VALUE,
114 'timeout': TIMEOUT_VALUE,
115 }
116 test = Exists(self.client, **kwargs)
117 test.wait()
119 # Update the name and run
120 logger.debug('Updating self.name from "%s" to "%s"...', self.name, newidx)
121 self.name = newidx
123 # Wait for the ILM steps to complete
124 logger.debug('Waiting for the ILM steps to complete...')
125 self._ilm_step()
127 # Track the new index
128 logger.debug('Switching to track "%s" as self.name...', newidx)
129 self.track_ilm(newidx)
131 # This is maybe unnecessary. This is for progressing ILM, e.g. from
132 # hot -> warm -> cold -> frozen (and even through delete).
133 #
134 # def _loop_until_target(self) -> None:
135 # current, target = self.phase_tuple
136 # while current != target:
137 # logger.debug(
138 # 'Attempting to move %s to ILM phase %s', self.name, target
139 # )
140 # self.ilm_tracker.advance(phase=target)
141 # # At this point, it's "in" a searchable tier, but the index name hasn't
142 # # changed yet
143 # newidx = mounted_name(self.name, target)
144 # logger.debug(
145 # 'Waiting for ILM phase change to complete. New index: %s', newidx
146 # )
147 # kwargs = {
148 # 'name': newidx,
149 # 'kind': 'index',
150 # 'pause': PAUSE_VALUE,
151 # 'timeout': TIMEOUT_VALUE,
152 # }
153 # test = Exists(self.client, **kwargs)
154 # test.wait()
155 # logger.info('ILM advance to phase %s completed', target)
156 # self.name = newidx
157 # self.track_ilm(self.name) # Refresh the ilm_tracker with the new name
158 # current, target = self.phase_tuple
160 def manual_ss(self, scheme) -> None:
161 """
162 If we are NOT using ILM but have specified searchable snapshots in the plan
163 entities
164 """
165 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']:
166 self.snapmgr.add(self.name, scheme['target_tier'])
167 # Replace self.name with the renamed name
168 self.name = mounted_name(self.name, scheme['target_tier'])
170 def mount_ss(self, scheme: dict) -> None:
171 """If the index is planned to become a searchable snapshot, we do that now"""
172 logger.debug('Checking if "%s" should be a searchable snapshot', self.name)
173 if self.am_i_write_idx:
174 logger.debug(
175 '"%s" is the write_index. Cannot mount as searchable snapshot',
176 self.name,
177 )
178 return
179 if not self.policy_name: # If we have this, chances are we have a policy
180 logger.debug('No ILM policy for "%s". Trying manual...', self.name)
181 self.manual_ss(scheme)
182 return
183 phase = self.ilm_tracker.next_phase
184 current = self.ilm_tracker.explain.phase
185 if current == 'new':
186 # This is a problem. We need to be in 'hot', with rollover completed.
187 logger.debug(
188 'Our index is still in phase "%s"!. We need it to be in "%s"',
189 current,
190 phase,
191 )
193 phasenext = IlmPhase(
194 self.client,
195 pause=PAUSE_VALUE,
196 timeout=TIMEOUT_VALUE,
197 name=self.name,
198 phase=self.ilm_tracker.next_phase,
199 )
200 phasenext.wait()
201 target = self._get_target
202 if current != target:
203 logger.debug('Current (%s) and target (%s) mismatch', current, target)
204 self.ilm_tracker.wait4complete()
205 # Because the step is completed, we must now update OUR tracker to
206 # reflect the updated ILM Explain information
207 self.ilm_tracker.update()
209 # ILM snapshot mount phase. The biggest pain of them all...
210 logger.debug('Moving "%s" to ILM phase "%s"', self.name, target)
211 self._mounted_step(target)
212 logger.info('ILM advance to phase "%s" completed', target)
214 # Record the snapshot in our tracker
215 self._add_snap_step()
217 def track_ilm(self, name: str) -> None:
218 """
219 Get ILM phase information and put it in self.ilm_tracker
220 Name as an arg makes it configurable
221 """
222 if self.policy_name:
223 self.ilm_tracker = IlmTracker(self.client, name)
224 self.ilm_tracker.update()