Coverage for curator/cli.py: 99%

138 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-16 15:27 -0600

1"""Main CLI for Curator""" 

2import os 

3import sys 

4import logging 

5import yaml 

6import click 

7from voluptuous import Schema 

8from curator import actions 

9from curator.config_utils import process_config, password_filter 

10from curator.defaults import settings 

11from curator.exceptions import ClientException, ConfigurationError, NoIndices, NoSnapshots 

12from curator.indexlist import IndexList 

13from curator.snapshotlist import SnapshotList 

14from curator.utils import get_client, get_yaml, prune_nones, validate_actions, get_write_index 

15from curator.validators import SchemaCheck 

16from curator._version import __version__ 

17 

18CLASS_MAP = { 

19 'alias' : actions.Alias, 

20 'allocation' : actions.Allocation, 

21 'close' : actions.Close, 

22 'cluster_routing' : actions.ClusterRouting, 

23 'create_index' : actions.CreateIndex, 

24 'delete_indices' : actions.DeleteIndices, 

25 'delete_snapshots' : actions.DeleteSnapshots, 

26 'forcemerge' : actions.ForceMerge, 

27 'index_settings' : actions.IndexSettings, 

28 'open' : actions.Open, 

29 'reindex' : actions.Reindex, 

30 'replicas' : actions.Replicas, 

31 'restore' : actions.Restore, 

32 'rollover' : actions.Rollover, 

33 'snapshot' : actions.Snapshot, 

34 'shrink' : actions.Shrink, 

35} 

36 

37def process_action(client, config, **kwargs): 

38 """ 

39 Do the `action` in the configuration dictionary, using the associated args. 

40 Other necessary args may be passed as keyword arguments 

41 

42 :arg config: An `action` dictionary. 

43 """ 

44 logger = logging.getLogger(__name__) 

45 # Make some placeholder variables here for readability 

46 logger.debug('Configuration dictionary: {0}'.format(config)) 

47 logger.debug('kwargs: {0}'.format(kwargs)) 

48 action = config['action'] 

49 # This will always have some defaults now, so no need to do the if... 

50 # # OLD WAY: opts = config['options'] if 'options' in config else {} 

51 opts = config['options'] 

52 logger.debug('opts: {0}'.format(opts)) 

53 mykwargs = {} 

54 

55 action_class = CLASS_MAP[action] 

56 

57 # Add some settings to mykwargs... 

58 if action == 'delete_indices': 

59 mykwargs['master_timeout'] = ( 

60 kwargs['master_timeout'] if 'master_timeout' in kwargs else 30) 

61 

62 ### Update the defaults with whatever came with opts, minus any Nones 

63 mykwargs.update(prune_nones(opts)) 

64 logger.debug('Action kwargs: {0}'.format(mykwargs)) 

65 

66 ### Set up the action ### 

67 if action == 'alias': 

68 # Special behavior for this action, as it has 2 index lists 

69 logger.debug('Running "{0}" action'.format(action.upper())) 

70 action_obj = action_class(**mykwargs) 

71 removes = IndexList(client) 

72 adds = IndexList(client) 

73 if 'remove' in config: 

74 logger.debug( 

75 'Removing indices from alias "{0}"'.format(opts['name'])) 

76 removes.iterate_filters(config['remove']) 

77 action_obj.remove( 

78 removes, warn_if_no_indices=opts['warn_if_no_indices']) 

79 if 'add' in config: 

80 logger.debug('Adding indices to alias "{0}"'.format(opts['name'])) 

81 adds.iterate_filters(config['add']) 

82 action_obj.add(adds, warn_if_no_indices=opts['warn_if_no_indices']) 

83 elif action in ['cluster_routing', 'create_index', 'rollover']: 

84 action_obj = action_class(client, **mykwargs) 

85 elif action == 'delete_snapshots' or action == 'restore': 

86 logger.debug('Running "{0}"'.format(action)) 

87 slo = SnapshotList(client, repository=opts['repository']) 

88 slo.iterate_filters(config) 

89 # We don't need to send this value to the action 

90 mykwargs.pop('repository') 

91 action_obj = action_class(slo, **mykwargs) 

92 else: 

93 logger.debug('Running "{0}"'.format(action.upper())) 

94 ilo = IndexList(client) 

95 ilo.iterate_filters(config) 

96 action_obj = action_class(ilo, **mykwargs) 

97 ### Do the action 

98 if 'dry_run' in kwargs and kwargs['dry_run']: 

99 action_obj.do_dry_run() 

100 else: 

101 logger.debug('Doing the action here.') 

102 action_obj.do_action() 

103 

104def run(config, action_file, dry_run=False): 

105 """ 

106 Actually run. 

107 """ 

108 client_args = process_config(config) 

109 logger = logging.getLogger(__name__) 

110 logger.debug('Client and logging options validated.') 

111 

112 # Extract this and save it for later, in case there's no timeout_override. 

113 default_timeout = client_args.pop('timeout') 

114 logger.debug('default_timeout = {0}'.format(default_timeout)) 

115 ######################################### 

116 ### Start working on the actions here ### 

117 ######################################### 

118 logger.debug('action_file: {0}'.format(action_file)) 

119 action_config = get_yaml(action_file) 

120 logger.debug('action_config: {0}'.format(password_filter(action_config))) 

121 action_dict = validate_actions(action_config) 

122 actions = action_dict['actions'] 

123 logger.debug('Full list of actions: {0}'.format(password_filter(actions))) 

124 action_keys = sorted(list(actions.keys())) 

125 for idx in action_keys: 

126 action = actions[idx]['action'] 

127 action_disabled = actions[idx]['options'].pop('disable_action') 

128 logger.debug('action_disabled = {0}'.format(action_disabled)) 

129 continue_if_exception = ( 

130 actions[idx]['options'].pop('continue_if_exception')) 

131 logger.debug( 

132 'continue_if_exception = {0}'.format(continue_if_exception)) 

133 timeout_override = actions[idx]['options'].pop('timeout_override') 

134 logger.debug('timeout_override = {0}'.format(timeout_override)) 

135 ignore_empty_list = actions[idx]['options'].pop('ignore_empty_list') 

136 logger.debug('ignore_empty_list = {0}'.format(ignore_empty_list)) 

137 allow_ilm = actions[idx]['options'].pop('allow_ilm_indices') 

138 logger.debug('allow_ilm_indices = {0}'.format(allow_ilm)) 

139 

140 ### Skip to next action if 'disabled' 

141 if action_disabled: 

142 logger.info( 

143 'Action ID: {0}: "{1}" not performed because "disable_action" ' 

144 'is set to True'.format(idx, action) 

145 ) 

146 continue 

147 else: 

148 logger.info('Preparing Action ID: {0}, "{1}"'.format(idx, action)) 

149 # Override the timeout, if specified, otherwise use the default. 

150 if isinstance(timeout_override, int): 

151 client_args['timeout'] = timeout_override 

152 else: 

153 client_args['timeout'] = default_timeout 

154 

155 # Set up action kwargs 

156 kwargs = {} 

157 kwargs['master_timeout'] = ( 

158 client_args['timeout'] if client_args['timeout'] <= 300 else 300) 

159 kwargs['dry_run'] = dry_run 

160 

161 # Create a client object for each action... 

162 logger.info('Creating client object and testing connection') 

163 try: 

164 client = get_client(**client_args) 

165 except (ClientException, ConfigurationError): 

166 sys.exit(1) 

167 ### Filter ILM indices unless expressly permitted 

168 if allow_ilm: 

169 logger.warning('allow_ilm_indices: true') 

170 logger.warning('Permitting operation on indices with an ILM policy') 

171 if not allow_ilm and action not in settings.snapshot_actions(): 

172 if actions[idx]['action'] == 'rollover': 

173 alias = actions[idx]['options']['name'] 

174 write_index = get_write_index(client, alias) 

175 try: 

176 idx_settings = client.indices.get_settings(index=write_index) 

177 if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']: 

178 logger.info('Alias {0} is associated with ILM policy.'.format(alias)) 

179 logger.info( 

180 'Skipping action {0} because allow_ilm_indices is false.'.format(idx)) 

181 continue 

182 except KeyError: 

183 logger.debug('No ILM policies associated with {0}'.format(alias)) 

184 elif 'filters' in actions[idx]: 

185 actions[idx]['filters'].append({'filtertype': 'ilm'}) 

186 else: 

187 actions[idx]['filters'] = [{'filtertype': 'ilm'}] 

188 ########################## 

189 ### Process the action ### 

190 ########################## 

191 try: 

192 logger.info( 

193 'Trying Action ID: {0}, "{1}": ' 

194 '{2}'.format(idx, action, actions[idx]['description']) 

195 ) 

196 process_action(client, actions[idx], **kwargs) 

197 except Exception as err: 

198 if isinstance(err, NoIndices) or isinstance(err, NoSnapshots): 

199 if ignore_empty_list: 

200 logger.info( 

201 'Skipping action "{0}" due to empty list: ' 

202 '{1}'.format(action, type(err)) 

203 ) 

204 else: 

205 logger.error( 

206 'Unable to complete action "{0}". No actionable items ' 

207 'in list: {1}'.format(action, type(err)) 

208 ) 

209 sys.exit(1) 

210 else: 

211 logger.error( 

212 'Failed to complete action: {0}. {1}: ' 

213 '{2}'.format(action, type(err), err) 

214 ) 

215 if continue_if_exception: 

216 logger.info( 

217 'Continuing execution with next action because ' 

218 '"continue_if_exception" is set to True for action ' 

219 '{0}'.format(action) 

220 ) 

221 else: 

222 sys.exit(1) 

223 logger.info('Action ID: {0}, "{1}" completed.'.format(idx, action)) 

224 logger.info('Job completed.') 

225 

226@click.command() 

227@click.option( 

228 '--config', 

229 help="Path to configuration file. Default: ~/.curator/curator.yml", 

230 type=click.Path(exists=True), default=settings.config_file() 

231) 

232@click.option('--dry-run', is_flag=True, help='Do not perform any changes.') 

233@click.argument('action_file', type=click.Path(exists=True), nargs=1) 

234@click.version_option(version=__version__) 

235def cli(config, dry_run, action_file): 

236 """ 

237 Curator for Elasticsearch indices. 

238 

239 See http://elastic.co/guide/en/elasticsearch/client/curator/current 

240 """ 

241 run(config, action_file, dry_run)