1
2 import os
3 import urllib
4 import ConfigParser
5
6 from starcluster import utils
7 from starcluster import static
8 from starcluster import awsutils
9 from starcluster import exception
10 from starcluster.cluster import Cluster
11 from starcluster.utils import AttributeDict
12
13 from starcluster.logger import log
14
15 DEBUG_CONFIG = False
19 """
20 Factory for EasyS3 class that attempts to load AWS credentials from
21 the StarCluster config file. Returns an EasyS3 object if
22 successful.
23 """
24 cfg = get_config(config_file, cache)
25 return cfg.get_easy_s3()
26
29 """
30 Factory for EasyEC2 class that attempts to load AWS credentials from
31 the StarCluster config file. Returns an EasyEC2 object if
32 successful.
33 """
34 cfg = get_config(config_file, cache)
35 return cfg.get_easy_ec2()
36
39 """Factory for StarClusterConfig object"""
40 return StarClusterConfig(config_file, cache)
41
44 """
45 Loads StarCluster configuration settings defined in config_file
46 which defaults to ~/.starclustercfg
47
48 Settings are available as follows:
49
50 cfg = StarClusterConfig()
51 or
52 cfg = StarClusterConfig('/path/to/my/config.cfg')
53 cfg.load()
54 aws_info = cfg.aws
55 cluster_cfg = cfg.clusters['mycluster']
56 key_cfg = cfg.keys['gsg-keypair']
57 print cluster_cfg
58 """
59
60 global_settings = static.GLOBAL_SETTINGS
61 aws_settings = static.AWS_SETTINGS
62 key_settings = static.KEY_SETTINGS
63 volume_settings = static.EBS_VOLUME_SETTINGS
64 plugin_settings = static.PLUGIN_SETTINGS
65 cluster_settings = static.CLUSTER_SETTINGS
66 permission_settings = static.PERMISSION_SETTINGS
67
68
69 instance_types = static.INSTANCE_TYPES
70
71 - def __init__(self, config_file=None, cache=False):
89
91 return "<StarClusterConfig: %s>" % self.cfg_file
92
94 log.debug("Loading url: %s" % url)
95 try:
96 fp = urllib.urlopen(url)
97 if fp.getcode() == 404:
98 raise exception.ConfigError("url %s does not exist" % url)
99 fp.name = url
100 return fp
101 except IOError, e:
102 raise exception.ConfigError(
103 "error loading config from url %s\n%s" % (url, e))
104
119
120 - def _get_bool(self, config, section, option):
121 try:
122 opt = config.getboolean(section, option)
123 return opt
124 except ConfigParser.NoSectionError:
125 pass
126 except ConfigParser.NoOptionError:
127 pass
128 except ValueError:
129 raise exception.ConfigError(
130 "Expected True/False value for setting %s in section [%s]" %
131 (option, section))
132
133 - def _get_int(self, config, section, option):
134 try:
135 opt = config.getint(section, option)
136 return opt
137 except ConfigParser.NoSectionError:
138 pass
139 except ConfigParser.NoOptionError:
140 pass
141 except ValueError:
142 raise exception.ConfigError(
143 "Expected integer value for setting %s in section [%s]" %
144 (option, section))
145
147 try:
148 opt = config.getfloat(section, option)
149 return opt
150 except ConfigParser.NoSectionError:
151 pass
152 except ConfigParser.NoOptionError:
153 pass
154 except ValueError:
155 raise exception.ConfigError(
156 "Expected float value for setting %s in section [%s]" %
157 (option, section))
158
160 try:
161 opt = config.get(section, option)
162 return opt
163 except ConfigParser.NoSectionError:
164 pass
165 except ConfigParser.NoOptionError:
166 pass
167
168 - def _get_list(self, config, section, option):
169 val = self._get_string(config, section, option)
170 if val:
171 val = [v.strip() for v in val.split(',')]
172 return val
173
191
193 """
194 Reloads the configuration file
195 """
196 self._config = self.__load_config()
197 return self.load()
198
199 @property
201 if self._config is None:
202 self._config = self.__load_config()
203 return self._config
204
205 - def _load_settings(self, section_name, settings, store,
206 filter_settings=True):
207 """
208 Load section settings into a dictionary
209 """
210 section = self.config._sections.get(section_name)
211 if not section:
212 raise exception.ConfigSectionMissing(
213 'Missing section %s in config' % section_name)
214 store.update(section)
215 section_conf = store
216 for setting in settings:
217 requirements = settings[setting]
218 func, required, default, options, callback = requirements
219 func = self.type_validators.get(func)
220 value = func(self.config, section_name, setting)
221 if value is not None:
222 if options and not value in options:
223 raise exception.ConfigError(
224 '"%s" setting in section "%s" must be one of: %s' %
225 (setting, section_name,
226 ', '.join([str(o) for o in options])))
227 if callback:
228 value = callback(value)
229 section_conf[setting] = value
230 if filter_settings:
231 for key in store.keys():
232 if key not in settings and key != '__name__':
233 store.pop(key)
234
236 """
237 Check that all required settings were specified in the config.
238 Raises ConfigError otherwise.
239
240 Note that if a setting specified has required=True and
241 default is not None then this method will not raise an error
242 because a default was given. In short, if a setting is required
243 you must provide None as the 'default' value.
244 """
245 section_conf = store
246 for setting in settings:
247 requirements = settings[setting]
248 required = requirements[1]
249 value = section_conf.get(setting)
250 if value is None and required:
251 raise exception.ConfigError(
252 'missing required option %s in section "%s"' %
253 (setting, section_name))
254
256 """
257 Sets the default for each setting in settings regardless of whether
258 the setting was specified in the config or not.
259 """
260 section_conf = store
261 for setting in settings:
262 default = settings[setting][2]
263 if section_conf.get(setting) is None:
264 if DEBUG_CONFIG:
265 log.debug('%s setting not specified. Defaulting to %s' % \
266 (setting, default))
267 section_conf[setting] = default
268
270 """
271 Loads all settings from other template(s) specified by a section's
272 'extends' setting.
273
274 This method walks a dependency tree of sections from bottom up. Each
275 step is a group of settings for a section in the form of a dictionary.
276 A 'master' dictionary is updated with the settings at each step. This
277 causes the next group of settings to override the previous, and so on.
278 The 'section_name' settings are at the top of the dep tree.
279 """
280 section = store[section_name]
281 extends = section.get('extends')
282 if extends is None:
283 return
284 if DEBUG_CONFIG:
285 log.debug('%s extends %s' % (section_name, extends))
286 extensions = [section]
287 while True:
288 extends = section.get('extends', None)
289 if not extends:
290 break
291 try:
292 section = store[extends]
293 if section in extensions:
294 exts = ', '.join([self._get_section_name(x['__name__'])
295 for x in extensions])
296 raise exception.ConfigError(
297 ("Cyclical dependency between sections %s. " % exts) \
298 + "Check your extends settings.")
299 extensions.insert(0, section)
300 except KeyError:
301 raise exception.ConfigError(
302 "%s can't extend non-existent section %s" % \
303 (section_name, extends))
304 transform = AttributeDict()
305 for extension in extensions:
306 transform.update(extension)
307 store[section_name] = transform
308
310 cluster_section = store
311 keyname = cluster_section.get('keyname')
312 if not keyname:
313 return
314 keypair = self.keys.get(keyname)
315 if keypair is None:
316 raise exception.ConfigError(
317 "keypair '%s' not defined in config" % keyname)
318 cluster_section['keyname'] = keyname
319 cluster_section['key_location'] = keypair.get('key_location')
320
334
336 cluster_section = store
337 plugins = cluster_section.get('plugins')
338 if not plugins or isinstance(plugins[0], AttributeDict):
339 return
340 plugs = []
341 cluster_section['plugins'] = plugs
342 for plugin in plugins:
343 if plugin in self.plugins:
344 p = self.plugins.get(plugin)
345 p['__name__'] = p['__name__'].split()[-1]
346 plugs.append(p)
347 else:
348 raise exception.ConfigError(
349 "plugin '%s' not defined in config" % plugin)
350
352 cluster_section = store
353 permissions = cluster_section.get('permissions')
354 if not permissions or isinstance(permissions, AttributeDict):
355 return
356 perms = AttributeDict()
357 cluster_section['permissions'] = perms
358 for perm in permissions:
359 if perm in self.permissions:
360 p = self.permissions.get(perm)
361 p['__name__'] = p['__name__'].split()[-1]
362 perms[perm] = p
363 else:
364 raise exception.ConfigError(
365 "permission '%s' not defined in config" % perm)
366
368 cluster_section = store
369 instance_types = cluster_section.get('node_instance_type')
370 if isinstance(instance_types, basestring):
371 return
372 itypes = []
373 cluster_section['node_instance_types'] = itypes
374 total_num_nodes = 0
375 choices_string = ', '.join(static.INSTANCE_TYPES.keys())
376 try:
377 default_instance_type = instance_types[-1]
378 if not default_instance_type in static.INSTANCE_TYPES:
379 raise exception.ConfigError(
380 ("invalid node_instance_type specified: '%s'\n" +
381 "must be one of: %s") %
382 (default_instance_type, choices_string))
383 except IndexError:
384 default_instance_type = None
385 cluster_section['node_instance_type'] = default_instance_type
386 for type_spec in instance_types[:-1]:
387 type_spec = type_spec.split(':')
388 if len(type_spec) > 3:
389 raise exception.ConfigError(
390 "invalid node_instance_type item specified: %s" % \
391 type_spec)
392 itype = type_spec[0]
393 itype_image = None
394 itype_num = 1
395 if not itype in static.INSTANCE_TYPES:
396 raise exception.ConfigError(
397 ("invalid type specified (%s) in node_instance_type " + \
398 "item: '%s'\nmust be one of: %s") %
399 (itype, type_spec, choices_string))
400 if len(type_spec) == 2:
401 itype, next_var = type_spec
402 try:
403 itype_num = int(next_var)
404 except (TypeError, ValueError):
405 itype_image = next_var
406 elif len(type_spec) == 3:
407 itype, itype_image, itype_num = type_spec
408 try:
409 itype_num = int(itype_num)
410 if itype_num < 1:
411 raise TypeError
412 total_num_nodes += itype_num
413 except (ValueError, TypeError):
414 raise exception.ConfigError(
415 ("number of instances (%s) of type '%s' must " + \
416 "be an integer > 1") % (itype_num, itype))
417 itype_dic = AttributeDict(size=itype_num, image=itype_image,
418 type=itype)
419 itypes.append(itype_dic)
420
421 - def _load_section(self, section_name, section_settings,
422 filter_settings=True):
423 """
424 Returns a dictionary containing all section_settings for a given
425 section_name by first loading the settings in the config, loading
426 the defaults for all settings not specified, and then checking
427 that all required options have been specified
428 """
429 store = AttributeDict()
430 self._load_settings(section_name, section_settings, store,
431 filter_settings)
432 self._load_defaults(section_settings, store)
433 self._check_required(section_name, section_settings, store)
434 return store
435
437 """
438 Returns section name minus prefix
439 e.g.
440 $ print self._get_section('cluster smallcluster')
441 $ smallcluster
442 """
443 return section.split()[1]
444
446 """
447 Returns all sections starting with section_prefix
448 e.g.
449 $ print self._get_sections('cluster')
450 $ ['cluster smallcluster', 'cluster mediumcluster', ..]
451 """
452 return [s for s in self.config.sections() if
453 s.startswith(section_prefix)]
454
455 - def _load_sections(self, section_prefix, section_settings,
456 filter_settings=True):
457 """
458 Loads all sections starting with section_prefix and returns a
459 dictionary containing the name and dictionary of settings for each
460 section.
461 keys --> section name (as returned by self._get_section_name)
462 values --> dictionary of settings for a given section
463
464 e.g.
465 $ print self._load_sections('volumes', self.plugin_settings)
466
467 {'myvol': {'__name__': 'volume myvol',
468 'device': None,
469 'mount_path': '/home',
470 'partition': 1,
471 'volume_id': 'vol-999999'},
472 'myvol2': {'__name__': 'volume myvol2',
473 'device': None,
474 'mount_path': '/myvol2',
475 'partition': 1,
476 'volume_id': 'vol-999999'},
477 """
478 sections = self._get_sections(section_prefix)
479 sections_store = AttributeDict()
480 for sec in sections:
481 name = self._get_section_name(sec)
482 sections_store[name] = self._load_section(sec, section_settings,
483 filter_settings)
484 return sections_store
485
487 """
488 Loads all cluster sections. Similar to _load_sections but also handles
489 populating specified keypair,volume,plugins,permissions,etc settings
490 """
491 clusters = cluster_sections
492 cluster_store = AttributeDict()
493 for cluster in clusters:
494 name = self._get_section_name(cluster)
495 cluster_store[name] = AttributeDict()
496 self._load_settings(cluster, self.cluster_settings,
497 cluster_store[name])
498 for cluster in clusters:
499 name = self._get_section_name(cluster)
500 self._load_extends_settings(name, cluster_store)
501 self._load_defaults(self.cluster_settings, cluster_store[name])
502 self._load_keypairs(cluster_store[name])
503 self._load_volumes(cluster_store[name])
504 self._load_plugins(cluster_store[name])
505 self._load_permissions(cluster_store[name])
506 self._load_instance_types(cluster_store[name])
507 self._check_required(cluster, self.cluster_settings,
508 cluster_store[name])
509 return cluster_store
510
535
537 """
538 Returns AWS credentials defined in the user's shell
539 environment.
540 """
541 awscreds = {}
542 for key in static.AWS_SETTINGS:
543 if key.upper() in os.environ:
544 awscreds[key] = os.environ.get(key.upper())
545 elif key in os.environ:
546 awscreds[key] = os.environ.get(key)
547 return awscreds
548
550 """
551 Returns AWS credentials defined in the configuration
552 file. Defining any of the AWS settings in the environment
553 overrides the configuration file.
554 """
555
556 self.aws.update(self.get_aws_from_environ())
557 return self.aws
558
561
564 """
565 Returns Cluster instance configured with the settings in the
566 config file.
567
568 template_name is the name of a cluster section defined in the config
569
570 tag_name, if specified, will be passed to Cluster instance
571 as cluster_tag
572 """
573 try:
574 kwargs = {}
575 if tag_name:
576 kwargs.update(dict(cluster_tag=tag_name))
577 kwargs.update(self.clusters[template_name])
578 if not ec2_conn:
579 ec2_conn = self.get_easy_ec2()
580 clust = Cluster(ec2_conn, **kwargs)
581 return clust
582 except KeyError:
583 raise exception.ClusterTemplateDoesNotExist(template_name)
584
586 """
587 Returns the default_template specified in the [global] section
588 of the config. Raises NoDefaultTemplateFound if no default cluster
589 template has been specified in the config.
590 """
591 default = self.globals.get('default_template')
592 if not default:
593 raise exception.NoDefaultTemplateFound(
594 options=self.clusters.keys())
595 if not default in self.clusters:
596 raise exception.ClusterTemplateDoesNotExist(default)
597 return default
598
604
610
612 try:
613 return self.keys[keyname]
614 except KeyError:
615 raise exception.KeyNotFound(keyname)
616
618 """
619 Factory for EasyEC2 class that attempts to load AWS credentials from
620 the StarCluster config file. Returns an EasyS3 object if
621 successful.
622 """
623 aws = self.get_aws_credentials()
624 try:
625 s3 = awsutils.EasyS3(**aws)
626 return s3
627 except TypeError:
628 raise exception.ConfigError("no aws credentials found")
629
631 """
632 Factory for EasyEC2 class that attempts to load AWS credentials from
633 the StarCluster config file. Returns an EasyEC2 object if
634 successful.
635 """
636 aws = self.get_aws_credentials()
637 try:
638 ec2 = awsutils.EasyEC2(**aws)
639 return ec2
640 except TypeError:
641 raise exception.ConfigError("no aws credentials found")
642
643 if __name__ == "__main__":
644 from pprint import pprint
645 cfg = StarClusterConfig().load()
646 pprint(cfg.aws)
647 pprint(cfg.clusters)
648 pprint(cfg.keys)
649 pprint(cfg.vols)
650