1
2 import os
3 import urllib
4 import ConfigParser
5
6 from starcluster import utils
7 from starcluster import static
8 from starcluster import cluster
9 from starcluster import awsutils
10 from starcluster import exception
11 from starcluster.cluster import Cluster
12 from starcluster.utils import AttributeDict
13
14 from starcluster.logger import log
15
16 DEBUG_CONFIG = False
20 """
21 Factory for EasyS3 class that attempts to load AWS credentials from
22 the StarCluster config file. Returns an EasyS3 object if
23 successful.
24 """
25 cfg = get_config(config_file, cache)
26 return cfg.get_easy_s3()
27
30 """
31 Factory for EasyEC2 class that attempts to load AWS credentials from
32 the StarCluster config file. Returns an EasyEC2 object if
33 successful.
34 """
35 cfg = get_config(config_file, cache)
36 return cfg.get_easy_ec2()
37
40 """
41 Factory for ClusterManager class that attempts to load AWS credentials from
42 the StarCluster config file. Returns a ClusterManager object if successful
43 """
44 cfg = get_config(config_file, cache)
45 return cfg.get_cluster_manager()
46
49 """Factory for StarClusterConfig object"""
50 return StarClusterConfig(config_file, cache)
51
54 """
55 Loads StarCluster configuration settings defined in config_file
56 which defaults to ~/.starclustercfg
57
58 Settings are available as follows:
59
60 cfg = StarClusterConfig()
61 or
62 cfg = StarClusterConfig('/path/to/my/config.cfg')
63 cfg.load()
64 aws_info = cfg.aws
65 cluster_cfg = cfg.clusters['mycluster']
66 key_cfg = cfg.keys['gsg-keypair']
67 print cluster_cfg
68 """
69
70 global_settings = static.GLOBAL_SETTINGS
71 aws_settings = static.AWS_SETTINGS
72 key_settings = static.KEY_SETTINGS
73 volume_settings = static.EBS_VOLUME_SETTINGS
74 plugin_settings = static.PLUGIN_SETTINGS
75 cluster_settings = static.CLUSTER_SETTINGS
76 permission_settings = static.PERMISSION_SETTINGS
77
78
79 instance_types = static.INSTANCE_TYPES
80
81 - def __init__(self, config_file=None, cache=False):
99
101 return "<StarClusterConfig: %s>" % self.cfg_file
102
104 log.debug("Loading url: %s" % url)
105 try:
106 fp = urllib.urlopen(url)
107 if fp.getcode() == 404:
108 raise exception.ConfigError("url %s does not exist" % url)
109 fp.name = url
110 return fp
111 except IOError, e:
112 raise exception.ConfigError(
113 "error loading config from url %s\n%s" % (url, e))
114
129
130 - def _get_bool(self, config, section, option):
131 try:
132 opt = config.getboolean(section, option)
133 return opt
134 except ConfigParser.NoSectionError:
135 pass
136 except ConfigParser.NoOptionError:
137 pass
138 except ValueError:
139 raise exception.ConfigError(
140 "Expected True/False value for setting %s in section [%s]" %
141 (option, section))
142
143 - def _get_int(self, config, section, option):
144 try:
145 opt = config.getint(section, option)
146 return opt
147 except ConfigParser.NoSectionError:
148 pass
149 except ConfigParser.NoOptionError:
150 pass
151 except ValueError:
152 raise exception.ConfigError(
153 "Expected integer value for setting %s in section [%s]" %
154 (option, section))
155
157 try:
158 opt = config.getfloat(section, option)
159 return opt
160 except ConfigParser.NoSectionError:
161 pass
162 except ConfigParser.NoOptionError:
163 pass
164 except ValueError:
165 raise exception.ConfigError(
166 "Expected float value for setting %s in section [%s]" %
167 (option, section))
168
170 try:
171 opt = config.get(section, option)
172 return opt
173 except ConfigParser.NoSectionError:
174 pass
175 except ConfigParser.NoOptionError:
176 pass
177
178 - def _get_list(self, config, section, option):
179 val = self._get_string(config, section, option)
180 if val:
181 val = [v.strip() for v in val.split(',')]
182 return val
183
201
203 """
204 Reloads the configuration file
205 """
206 self._config = self.__load_config()
207 return self.load()
208
209 @property
211 if self._config is None:
212 self._config = self.__load_config()
213 return self._config
214
215 - def _load_settings(self, section_name, settings, store,
216 filter_settings=True):
217 """
218 Load section settings into a dictionary
219 """
220 section = self.config._sections.get(section_name)
221 if not section:
222 raise exception.ConfigSectionMissing(
223 'Missing section %s in config' % section_name)
224 store.update(section)
225 section_conf = store
226 for setting in settings:
227 requirements = settings[setting]
228 func, required, default, options, callback = requirements
229 func = self.type_validators.get(func)
230 value = func(self.config, section_name, setting)
231 if value is not None:
232 if options and not value in options:
233 raise exception.ConfigError(
234 '"%s" setting in section "%s" must be one of: %s' %
235 (setting, section_name,
236 ', '.join([str(o) for o in options])))
237 if callback:
238 value = callback(value)
239 section_conf[setting] = value
240 if filter_settings:
241 for key in store.keys():
242 if key not in settings and key != '__name__':
243 store.pop(key)
244
246 """
247 Check that all required settings were specified in the config.
248 Raises ConfigError otherwise.
249
250 Note that if a setting specified has required=True and
251 default is not None then this method will not raise an error
252 because a default was given. In short, if a setting is required
253 you must provide None as the 'default' value.
254 """
255 section_conf = store
256 for setting in settings:
257 requirements = settings[setting]
258 required = requirements[1]
259 value = section_conf.get(setting)
260 if value is None and required:
261 raise exception.ConfigError(
262 'missing required option %s in section "%s"' %
263 (setting, section_name))
264
266 """
267 Sets the default for each setting in settings regardless of whether
268 the setting was specified in the config or not.
269 """
270 section_conf = store
271 for setting in settings:
272 default = settings[setting][2]
273 if section_conf.get(setting) is None:
274 if DEBUG_CONFIG:
275 log.debug('%s setting not specified. Defaulting to %s' % \
276 (setting, default))
277 section_conf[setting] = default
278
280 """
281 Loads all settings from other template(s) specified by a section's
282 'extends' setting.
283
284 This method walks a dependency tree of sections from bottom up. Each
285 step is a group of settings for a section in the form of a dictionary.
286 A 'master' dictionary is updated with the settings at each step. This
287 causes the next group of settings to override the previous, and so on.
288 The 'section_name' settings are at the top of the dep tree.
289 """
290 section = store[section_name]
291 extends = section.get('extends')
292 if extends is None:
293 return
294 if DEBUG_CONFIG:
295 log.debug('%s extends %s' % (section_name, extends))
296 extensions = [section]
297 while True:
298 extends = section.get('extends', None)
299 if not extends:
300 break
301 try:
302 section = store[extends]
303 if section in extensions:
304 exts = ', '.join([self._get_section_name(x['__name__'])
305 for x in extensions])
306 raise exception.ConfigError(
307 ("Cyclical dependency between sections %s. " % exts) \
308 + "Check your extends settings.")
309 extensions.insert(0, section)
310 except KeyError:
311 raise exception.ConfigError(
312 "%s can't extend non-existent section %s" % \
313 (section_name, extends))
314 transform = AttributeDict()
315 for extension in extensions:
316 transform.update(extension)
317 store[section_name] = transform
318
320 cluster_section = store
321 keyname = cluster_section.get('keyname')
322 if not keyname:
323 return
324 keypair = self.keys.get(keyname)
325 if keypair is None:
326 raise exception.ConfigError(
327 "keypair '%s' not defined in config" % keyname)
328 cluster_section['keyname'] = keyname
329 cluster_section['key_location'] = keypair.get('key_location')
330
344
346 cluster_section = store
347 plugins = cluster_section.get('plugins')
348 if not plugins or isinstance(plugins[0], AttributeDict):
349 return
350 plugs = []
351 cluster_section['plugins'] = plugs
352 for plugin in plugins:
353 if plugin in self.plugins:
354 p = self.plugins.get(plugin)
355 p['__name__'] = p['__name__'].split()[-1]
356 plugs.append(p)
357 else:
358 raise exception.ConfigError(
359 "plugin '%s' not defined in config" % plugin)
360
362 cluster_section = store
363 permissions = cluster_section.get('permissions')
364 if not permissions or isinstance(permissions, AttributeDict):
365 return
366 perms = AttributeDict()
367 cluster_section['permissions'] = perms
368 for perm in permissions:
369 if perm in self.permissions:
370 p = self.permissions.get(perm)
371 p['__name__'] = p['__name__'].split()[-1]
372 perms[perm] = p
373 else:
374 raise exception.ConfigError(
375 "permission '%s' not defined in config" % perm)
376
378 cluster_section = store
379 instance_types = cluster_section.get('node_instance_type')
380 if isinstance(instance_types, basestring):
381 return
382 itypes = []
383 cluster_section['node_instance_types'] = itypes
384 total_num_nodes = 0
385 choices_string = ', '.join(static.INSTANCE_TYPES.keys())
386 try:
387 default_instance_type = instance_types[-1]
388 if not default_instance_type in static.INSTANCE_TYPES:
389 raise exception.ConfigError(
390 ("invalid node_instance_type specified: '%s'\n" +
391 "must be one of: %s") %
392 (default_instance_type, choices_string))
393 except IndexError:
394 default_instance_type = None
395 cluster_section['node_instance_type'] = default_instance_type
396 for type_spec in instance_types[:-1]:
397 type_spec = type_spec.split(':')
398 if len(type_spec) > 3:
399 raise exception.ConfigError(
400 "invalid node_instance_type item specified: %s" % \
401 type_spec)
402 itype = type_spec[0]
403 itype_image = None
404 itype_num = 1
405 if not itype in static.INSTANCE_TYPES:
406 raise exception.ConfigError(
407 ("invalid type specified (%s) in node_instance_type " + \
408 "item: '%s'\nmust be one of: %s") %
409 (itype, type_spec, choices_string))
410 if len(type_spec) == 2:
411 itype, next_var = type_spec
412 try:
413 itype_num = int(next_var)
414 except (TypeError, ValueError):
415 itype_image = next_var
416 elif len(type_spec) == 3:
417 itype, itype_image, itype_num = type_spec
418 try:
419 itype_num = int(itype_num)
420 if itype_num < 1:
421 raise TypeError
422 total_num_nodes += itype_num
423 except (ValueError, TypeError):
424 raise exception.ConfigError(
425 ("number of instances (%s) of type '%s' must " + \
426 "be an integer > 1") % (itype_num, itype))
427 itype_dic = AttributeDict(size=itype_num, image=itype_image,
428 type=itype)
429 itypes.append(itype_dic)
430
431 - def _load_section(self, section_name, section_settings,
432 filter_settings=True):
433 """
434 Returns a dictionary containing all section_settings for a given
435 section_name by first loading the settings in the config, loading
436 the defaults for all settings not specified, and then checking
437 that all required options have been specified
438 """
439 store = AttributeDict()
440 self._load_settings(section_name, section_settings, store,
441 filter_settings)
442 self._load_defaults(section_settings, store)
443 self._check_required(section_name, section_settings, store)
444 return store
445
447 """
448 Returns section name minus prefix
449 e.g.
450 $ print self._get_section('cluster smallcluster')
451 $ smallcluster
452 """
453 return section.split()[1]
454
456 """
457 Returns all sections starting with section_prefix
458 e.g.
459 $ print self._get_sections('cluster')
460 $ ['cluster smallcluster', 'cluster mediumcluster', ..]
461 """
462 return [s for s in self.config.sections() if
463 s.startswith(section_prefix)]
464
465 - def _load_sections(self, section_prefix, section_settings,
466 filter_settings=True):
467 """
468 Loads all sections starting with section_prefix and returns a
469 dictionary containing the name and dictionary of settings for each
470 section.
471 keys --> section name (as returned by self._get_section_name)
472 values --> dictionary of settings for a given section
473
474 e.g.
475 $ print self._load_sections('volumes', self.plugin_settings)
476
477 {'myvol': {'__name__': 'volume myvol',
478 'device': None,
479 'mount_path': '/home',
480 'partition': 1,
481 'volume_id': 'vol-999999'},
482 'myvol2': {'__name__': 'volume myvol2',
483 'device': None,
484 'mount_path': '/myvol2',
485 'partition': 1,
486 'volume_id': 'vol-999999'},
487 """
488 sections = self._get_sections(section_prefix)
489 sections_store = AttributeDict()
490 for sec in sections:
491 name = self._get_section_name(sec)
492 sections_store[name] = self._load_section(sec, section_settings,
493 filter_settings)
494 return sections_store
495
497 """
498 Loads all cluster sections. Similar to _load_sections but also handles
499 populating specified keypair,volume,plugins,permissions,etc settings
500 """
501 clusters = cluster_sections
502 cluster_store = AttributeDict()
503 for cl in clusters:
504 name = self._get_section_name(cl)
505 cluster_store[name] = AttributeDict()
506 self._load_settings(cl, self.cluster_settings, cluster_store[name])
507 for cl in clusters:
508 name = self._get_section_name(cl)
509 self._load_extends_settings(name, cluster_store)
510 self._load_defaults(self.cluster_settings, cluster_store[name])
511 self._load_keypairs(cluster_store[name])
512 self._load_volumes(cluster_store[name])
513 self._load_plugins(cluster_store[name])
514 self._load_permissions(cluster_store[name])
515 self._load_instance_types(cluster_store[name])
516 self._check_required(cl, self.cluster_settings,
517 cluster_store[name])
518 return cluster_store
519
544
546 """
547 Returns AWS credentials defined in the user's shell
548 environment.
549 """
550 awscreds = {}
551 for key in static.AWS_SETTINGS:
552 if key.upper() in os.environ:
553 awscreds[key] = os.environ.get(key.upper())
554 elif key in os.environ:
555 awscreds[key] = os.environ.get(key)
556 return awscreds
557
559 """
560 Returns AWS credentials defined in the configuration
561 file. Defining any of the AWS settings in the environment
562 overrides the configuration file.
563 """
564
565 self.aws.update(self.get_aws_from_environ())
566 return self.aws
567
570
573 """
574 Returns Cluster instance configured with the settings in the
575 config file.
576
577 template_name is the name of a cluster section defined in the config
578
579 tag_name, if specified, will be passed to Cluster instance
580 as cluster_tag
581 """
582 try:
583 kwargs = {}
584 if tag_name:
585 kwargs.update(dict(cluster_tag=tag_name))
586 kwargs.update(self.clusters[template_name])
587 if not ec2_conn:
588 ec2_conn = self.get_easy_ec2()
589 clust = Cluster(ec2_conn, **kwargs)
590 return clust
591 except KeyError:
592 raise exception.ClusterTemplateDoesNotExist(template_name)
593
595 """
596 Returns the default_template specified in the [global] section
597 of the config. Raises NoDefaultTemplateFound if no default cluster
598 template has been specified in the config.
599 """
600 default = self.globals.get('default_template')
601 if not default:
602 raise exception.NoDefaultTemplateFound(
603 options=self.clusters.keys())
604 if not default in self.clusters:
605 raise exception.ClusterTemplateDoesNotExist(default)
606 return default
607
613
619
621 try:
622 return self.keys[keyname]
623 except KeyError:
624 raise exception.KeyNotFound(keyname)
625
627 """
628 Factory for EasyEC2 class that attempts to load AWS credentials from
629 the StarCluster config file. Returns an EasyS3 object if
630 successful.
631 """
632 aws = self.get_aws_credentials()
633 try:
634 s3 = awsutils.EasyS3(**aws)
635 return s3
636 except TypeError:
637 raise exception.ConfigError("no aws credentials found")
638
640 """
641 Factory for EasyEC2 class that attempts to load AWS credentials from
642 the StarCluster config file. Returns an EasyEC2 object if
643 successful.
644 """
645 aws = self.get_aws_credentials()
646 try:
647 ec2 = awsutils.EasyEC2(**aws)
648 return ec2
649 except TypeError:
650 raise exception.ConfigError("no aws credentials found")
651
655
656
657 if __name__ == "__main__":
658 from pprint import pprint
659 cfg = StarClusterConfig().load()
660 pprint(cfg.aws)
661 pprint(cfg.clusters)
662 pprint(cfg.keys)
663 pprint(cfg.vols)
664