Coverage for curator/actions/cold2frozen.py: 65%
75 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-20 21:00 -0600
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-20 21:00 -0600
1"""Snapshot and Restore action classes"""
2import logging
3from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference
4from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list
5from curator.helpers.utils import report_failure
6from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException
8class Cold2Frozen:
9 """Cold to Frozen Tier Searchable Snapshot Action Class
11 For manually migrating snapshots not associated with ILM from the cold tier to the frozen tier.
12 """
14 DEFAULTS = {
15 'index_settings': None,
16 'ignore_index_settings': ['index.refresh_interval'],
17 'wait_for_completion': True,
18 }
19 def __init__(self, ilo, **kwargs):
20 """
21 :param ilo: An IndexList Object
22 :param index_settings: (Optional) Settings that should be added to the index when it is
23 mounted. If not set, set the ``_tier_preference`` to the tiers available, coldest
24 first.
25 :param ignore_index_settings: (Optional, array of strings) Names of settings that should
26 be removed from the index when it is mounted.
27 :param wait_for_completion: Wait for completion before returning.
29 :type ilo: :py:class:`~.curator.indexlist.IndexList`
30 :type index_settings: dict
31 :type ignore_index_settings: list
32 :type wait_for_completion: bool
33 """
34 self.loggit = logging.getLogger('curator.actions.cold2frozen')
35 verify_index_list(ilo)
36 # Check here and don't bother with the rest of this if there are no
37 # indices in the index list.
38 ilo.empty_list_check()
40 #: The :py:class:`~.curator.indexlist.IndexList` object passed from param ``ilo``
41 self.index_list = ilo
42 #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from
43 #: :py:attr:`index_list`
44 self.client = ilo.client
45 #: Object attribute that contains the :py:func:`~.curator.helpers.utils.to_csv` output of
46 #: the indices in :py:attr:`index_list`.
47 self.indices = ilo
48 #: Object attribute that gets the value of ``index_settings``.
49 self.index_settings = None
50 #: Object attribute that gets the value of ``ignore_index_settings``.
51 self.ignore_index_settings = None
52 #: Object attribute that gets the value of param ``wait_for_completion``.
53 self.wait_for_completion = None
55 # Parse the kwargs into attributes
56 self.assign_kwargs(**kwargs)
58 def assign_kwargs(self, **kwargs):
59 """
60 Assign the kwargs to the attribute of the same name with the passed value or the default
61 from DEFAULTS
62 """
63 # Handy little loop here only adds kwargs that exist in DEFAULTS, or the default value.
64 # It ignores any non-relevant kwargs
65 for key, value in self.DEFAULTS.items():
66 if key in kwargs:
67 setattr(self, key, kwargs[key])
68 else:
69 setattr(self, key, value)
71 def action_generator(self):
72 """Yield a dict for use in :py:meth:`do_action` and :py:meth:`do_dry_run`
74 :returns: A generator object containing the settings necessary to migrate indices from cold
75 to frozen
76 :rtype: dict
77 """
78 for idx in self.index_list.indices:
79 idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index']
80 if has_lifecycle_name(idx_settings):
81 self.loggit.critical(
82 'Index %s is associated with an ILM policy and this action will never work on '
83 'an index associated with an ILM policy', idx)
84 raise CuratorException(f'Index {idx} is associated with an ILM policy')
85 if is_idx_partial(idx_settings):
86 self.loggit.critical('Index %s is already in the frozen tier', idx)
87 raise SearchableSnapshotException('Index is already in frozen tier')
88 snap = idx_settings['store']['snapshot']['snapshot_name']
89 snap_idx = idx_settings['store']['snapshot']['index_name']
90 repo = idx_settings['store']['snapshot']['repository_name']
91 aliases = self.client.indices.get(index=idx)[idx]['aliases']
93 prefix = get_frozen_prefix(snap_idx, idx)
94 renamed = f'{prefix}{snap_idx}'
96 if not self.index_settings:
97 self.index_settings = {
98 "routing": {
99 "allocation": {
100 "include": {
101 "_tier_preference": get_tier_preference(self.client)
102 }
103 }
104 }
105 }
106 yield {
107 'repository': repo, 'snapshot': snap, 'index': snap_idx,
108 'renamed_index': renamed, 'index_settings': self.index_settings,
109 'ignore_index_settings': self.ignore_index_settings,
110 'storage': 'shared_cache', 'wait_for_completion': self.wait_for_completion,
111 'aliases': aliases, 'current_idx': idx
112 }
114 def do_dry_run(self):
115 """Log what the output would be, but take no action."""
116 self.loggit.info('DRY-RUN MODE. No changes will be made.')
117 for kwargs in self.action_generator():
118 aliases = kwargs.pop('aliases')
119 current_idx = kwargs.pop('current_idx')
120 msg = (
121 f'DRY-RUN: cold2frozen: from snapshot {kwargs["snapshot"]} in repository '
122 f'{kwargs["repository"]}, mount index {kwargs["index"]} renamed as '
123 f'{kwargs["renamed_index"]} with index settings: {kwargs["index_settings"]} '
124 f'and ignoring settings: {kwargs["ignore_index_settings"]}. wait_for_completion: '
125 f'{kwargs["wait_for_completion"]}. Restore aliases: {aliases}. Current index '
126 f'name: {current_idx}'
127 )
128 self.loggit.info(msg)
131 def do_action(self):
132 """
133 Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices
134 in :py:attr:`ilo` in the Frozen tier.
136 Verify index looks good
138 Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new
139 frozen index with the aliases from the old cold-tier index.
141 Verify aliases look good.
143 Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index.
144 """
145 try:
146 for kwargs in self.action_generator():
147 aliases = kwargs.pop('aliases')
148 current_idx = kwargs.pop('current_idx')
149 newidx = kwargs['renamed_index']
150 # Actually do the mount
151 self.loggit.debug('Mounting new index %s in frozen tier...', newidx)
152 self.client.searchable_snapshots.mount(**kwargs)
153 # Verify it's mounted as a partial now:
154 self.loggit.debug('Verifying new index %s is mounted properly...', newidx)
155 idx_settings = self.client.indices.get(index=newidx)[newidx]
156 if is_idx_partial(idx_settings['settings']['index']):
157 self.loggit.info('Index %s is mounted for frozen tier', newidx)
158 else:
159 raise SearchableSnapshotException(
160 f'Index {newidx} not a mounted searchable snapshot')
161 # Update Aliases
162 alias_names = aliases.keys()
163 if not alias_names:
164 self.loggit.warning('No aliases associated with index %s', current_idx)
165 else:
166 self.loggit.debug('Transferring aliases to new index %s', newidx)
167 self.client.indices.update_aliases(
168 actions=get_alias_actions(current_idx, newidx, aliases))
169 verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys()
170 if alias_names != verify:
171 self.loggit.error(
172 'Alias names do not match! %s does not match: %s', alias_names, verify)
173 raise FailedExecution('Aliases failed to transfer to new index')
174 # Clean up old index
175 self.loggit.debug('Deleting old index: %s', current_idx)
176 self.client.indices.delete(index=current_idx)
177 self.loggit.info(
178 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx)
180 # pylint: disable=broad-except
181 except Exception as err:
182 report_failure(err)