Coverage for curator/actions/cold2frozen.py: 65%

75 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-20 21:00 -0600

1"""Snapshot and Restore action classes""" 

2import logging 

3from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference 

4from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list 

5from curator.helpers.utils import report_failure 

6from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException 

7 

8class Cold2Frozen: 

9 """Cold to Frozen Tier Searchable Snapshot Action Class 

10 

11 For manually migrating snapshots not associated with ILM from the cold tier to the frozen tier. 

12 """ 

13 

14 DEFAULTS = { 

15 'index_settings': None, 

16 'ignore_index_settings': ['index.refresh_interval'], 

17 'wait_for_completion': True, 

18 } 

19 def __init__(self, ilo, **kwargs): 

20 """ 

21 :param ilo: An IndexList Object 

22 :param index_settings: (Optional) Settings that should be added to the index when it is 

23 mounted. If not set, set the ``_tier_preference`` to the tiers available, coldest 

24 first. 

25 :param ignore_index_settings: (Optional, array of strings) Names of settings that should 

26 be removed from the index when it is mounted. 

27 :param wait_for_completion: Wait for completion before returning. 

28 

29 :type ilo: :py:class:`~.curator.indexlist.IndexList` 

30 :type index_settings: dict 

31 :type ignore_index_settings: list 

32 :type wait_for_completion: bool 

33 """ 

34 self.loggit = logging.getLogger('curator.actions.cold2frozen') 

35 verify_index_list(ilo) 

36 # Check here and don't bother with the rest of this if there are no 

37 # indices in the index list. 

38 ilo.empty_list_check() 

39 

40 #: The :py:class:`~.curator.indexlist.IndexList` object passed from param ``ilo`` 

41 self.index_list = ilo 

42 #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from 

43 #: :py:attr:`index_list` 

44 self.client = ilo.client 

45 #: Object attribute that contains the :py:func:`~.curator.helpers.utils.to_csv` output of 

46 #: the indices in :py:attr:`index_list`. 

47 self.indices = ilo 

48 #: Object attribute that gets the value of ``index_settings``. 

49 self.index_settings = None 

50 #: Object attribute that gets the value of ``ignore_index_settings``. 

51 self.ignore_index_settings = None 

52 #: Object attribute that gets the value of param ``wait_for_completion``. 

53 self.wait_for_completion = None 

54 

55 # Parse the kwargs into attributes 

56 self.assign_kwargs(**kwargs) 

57 

58 def assign_kwargs(self, **kwargs): 

59 """ 

60 Assign the kwargs to the attribute of the same name with the passed value or the default 

61 from DEFAULTS 

62 """ 

63 # Handy little loop here only adds kwargs that exist in DEFAULTS, or the default value. 

64 # It ignores any non-relevant kwargs 

65 for key, value in self.DEFAULTS.items(): 

66 if key in kwargs: 

67 setattr(self, key, kwargs[key]) 

68 else: 

69 setattr(self, key, value) 

70 

71 def action_generator(self): 

72 """Yield a dict for use in :py:meth:`do_action` and :py:meth:`do_dry_run` 

73 

74 :returns: A generator object containing the settings necessary to migrate indices from cold 

75 to frozen 

76 :rtype: dict 

77 """ 

78 for idx in self.index_list.indices: 

79 idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index'] 

80 if has_lifecycle_name(idx_settings): 

81 self.loggit.critical( 

82 'Index %s is associated with an ILM policy and this action will never work on ' 

83 'an index associated with an ILM policy', idx) 

84 raise CuratorException(f'Index {idx} is associated with an ILM policy') 

85 if is_idx_partial(idx_settings): 

86 self.loggit.critical('Index %s is already in the frozen tier', idx) 

87 raise SearchableSnapshotException('Index is already in frozen tier') 

88 snap = idx_settings['store']['snapshot']['snapshot_name'] 

89 snap_idx = idx_settings['store']['snapshot']['index_name'] 

90 repo = idx_settings['store']['snapshot']['repository_name'] 

91 aliases = self.client.indices.get(index=idx)[idx]['aliases'] 

92 

93 prefix = get_frozen_prefix(snap_idx, idx) 

94 renamed = f'{prefix}{snap_idx}' 

95 

96 if not self.index_settings: 

97 self.index_settings = { 

98 "routing": { 

99 "allocation": { 

100 "include": { 

101 "_tier_preference": get_tier_preference(self.client) 

102 } 

103 } 

104 } 

105 } 

106 yield { 

107 'repository': repo, 'snapshot': snap, 'index': snap_idx, 

108 'renamed_index': renamed, 'index_settings': self.index_settings, 

109 'ignore_index_settings': self.ignore_index_settings, 

110 'storage': 'shared_cache', 'wait_for_completion': self.wait_for_completion, 

111 'aliases': aliases, 'current_idx': idx 

112 } 

113 

114 def do_dry_run(self): 

115 """Log what the output would be, but take no action.""" 

116 self.loggit.info('DRY-RUN MODE. No changes will be made.') 

117 for kwargs in self.action_generator(): 

118 aliases = kwargs.pop('aliases') 

119 current_idx = kwargs.pop('current_idx') 

120 msg = ( 

121 f'DRY-RUN: cold2frozen: from snapshot {kwargs["snapshot"]} in repository ' 

122 f'{kwargs["repository"]}, mount index {kwargs["index"]} renamed as ' 

123 f'{kwargs["renamed_index"]} with index settings: {kwargs["index_settings"]} ' 

124 f'and ignoring settings: {kwargs["ignore_index_settings"]}. wait_for_completion: ' 

125 f'{kwargs["wait_for_completion"]}. Restore aliases: {aliases}. Current index ' 

126 f'name: {current_idx}' 

127 ) 

128 self.loggit.info(msg) 

129 

130 

131 def do_action(self): 

132 """ 

133 Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices 

134 in :py:attr:`ilo` in the Frozen tier. 

135 

136 Verify index looks good 

137 

138 Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new 

139 frozen index with the aliases from the old cold-tier index. 

140 

141 Verify aliases look good. 

142 

143 Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index. 

144 """ 

145 try: 

146 for kwargs in self.action_generator(): 

147 aliases = kwargs.pop('aliases') 

148 current_idx = kwargs.pop('current_idx') 

149 newidx = kwargs['renamed_index'] 

150 # Actually do the mount 

151 self.loggit.debug('Mounting new index %s in frozen tier...', newidx) 

152 self.client.searchable_snapshots.mount(**kwargs) 

153 # Verify it's mounted as a partial now: 

154 self.loggit.debug('Verifying new index %s is mounted properly...', newidx) 

155 idx_settings = self.client.indices.get(index=newidx)[newidx] 

156 if is_idx_partial(idx_settings['settings']['index']): 

157 self.loggit.info('Index %s is mounted for frozen tier', newidx) 

158 else: 

159 raise SearchableSnapshotException( 

160 f'Index {newidx} not a mounted searchable snapshot') 

161 # Update Aliases 

162 alias_names = aliases.keys() 

163 if not alias_names: 

164 self.loggit.warning('No aliases associated with index %s', current_idx) 

165 else: 

166 self.loggit.debug('Transferring aliases to new index %s', newidx) 

167 self.client.indices.update_aliases( 

168 actions=get_alias_actions(current_idx, newidx, aliases)) 

169 verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys() 

170 if alias_names != verify: 

171 self.loggit.error( 

172 'Alias names do not match! %s does not match: %s', alias_names, verify) 

173 raise FailedExecution('Aliases failed to transfer to new index') 

174 # Clean up old index 

175 self.loggit.debug('Deleting old index: %s', current_idx) 

176 self.client.indices.delete(index=current_idx) 

177 self.loggit.info( 

178 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx) 

179 

180 # pylint: disable=broad-except 

181 except Exception as err: 

182 report_failure(err)