1
2 import os
3 import urllib
4 import StringIO
5 import ConfigParser
6
7 from starcluster import utils
8 from starcluster import static
9 from starcluster import cluster
10 from starcluster import awsutils
11 from starcluster import exception
12 from starcluster.cluster import Cluster
13 from starcluster.utils import AttributeDict
14
15 from starcluster.logger import log
16
17 DEBUG_CONFIG = False
21 """
22 Factory for EasyS3 class that attempts to load AWS credentials from
23 the StarCluster config file. Returns an EasyS3 object if
24 successful.
25 """
26 cfg = get_config(config_file, cache)
27 return cfg.get_easy_s3()
28
31 """
32 Factory for EasyEC2 class that attempts to load AWS credentials from
33 the StarCluster config file. Returns an EasyEC2 object if
34 successful.
35 """
36 cfg = get_config(config_file, cache)
37 return cfg.get_easy_ec2()
38
41 """
42 Factory for ClusterManager class that attempts to load AWS credentials from
43 the StarCluster config file. Returns a ClusterManager object if successful
44 """
45 cfg = get_config(config_file, cache)
46 return cfg.get_cluster_manager()
47
50 """Factory for StarClusterConfig object"""
51 return StarClusterConfig(config_file, cache)
52
55 """
56 Loads StarCluster configuration settings defined in config_file
57 which defaults to ~/.starclustercfg
58
59 Settings are available as follows:
60
61 cfg = StarClusterConfig()
62 or
63 cfg = StarClusterConfig('/path/to/my/config.cfg')
64 cfg.load()
65 aws_info = cfg.aws
66 cluster_cfg = cfg.clusters['mycluster']
67 key_cfg = cfg.keys['gsg-keypair']
68 print cluster_cfg
69 """
70
71 global_settings = static.GLOBAL_SETTINGS
72 aws_settings = static.AWS_SETTINGS
73 key_settings = static.KEY_SETTINGS
74 volume_settings = static.EBS_VOLUME_SETTINGS
75 plugin_settings = static.PLUGIN_SETTINGS
76 cluster_settings = static.CLUSTER_SETTINGS
77 permission_settings = static.PERMISSION_SETTINGS
78
79
80 instance_types = static.INSTANCE_TYPES
81
82 - def __init__(self, config_file=None, cache=False):
102
104 return "<StarClusterConfig: %s>" % self.cfg_file
105
107 log.debug("Loading url: %s" % url)
108 try:
109 fp = urllib.urlopen(url)
110 if fp.getcode() == 404:
111 raise exception.ConfigError("url %s does not exist" % url)
112 fp.name = url
113 return fp
114 except IOError, e:
115 raise exception.ConfigError(
116 "error loading config from url %s\n%s" % (url, e))
117
119 log.debug("Loading file: %s" % cfg_file)
120 if os.path.exists(cfg_file):
121 if not os.path.isfile(cfg_file):
122 raise exception.ConfigError(
123 'config %s exists but is not a regular file' % cfg_file)
124 else:
125 raise exception.ConfigNotFound("config file %s does not exist\n" %
126 cfg_file, cfg_file)
127 return open(cfg_file)
128
135
136 - def _get_bool(self, config, section, option):
137 try:
138 opt = config.getboolean(section, option)
139 return opt
140 except ConfigParser.NoSectionError:
141 pass
142 except ConfigParser.NoOptionError:
143 pass
144 except ValueError:
145 raise exception.ConfigError(
146 "Expected True/False value for setting %s in section [%s]" %
147 (option, section))
148
149 - def _get_int(self, config, section, option):
150 try:
151 opt = config.getint(section, option)
152 return opt
153 except ConfigParser.NoSectionError:
154 pass
155 except ConfigParser.NoOptionError:
156 pass
157 except ValueError:
158 raise exception.ConfigError(
159 "Expected integer value for setting %s in section [%s]" %
160 (option, section))
161
163 try:
164 opt = config.getfloat(section, option)
165 return opt
166 except ConfigParser.NoSectionError:
167 pass
168 except ConfigParser.NoOptionError:
169 pass
170 except ValueError:
171 raise exception.ConfigError(
172 "Expected float value for setting %s in section [%s]" %
173 (option, section))
174
176 try:
177 opt = config.get(section, option)
178 return opt
179 except ConfigParser.NoSectionError:
180 pass
181 except ConfigParser.NoOptionError:
182 pass
183
184 - def _get_list(self, config, section, option):
185 val = self._get_string(config, section, option)
186 if val:
187 val = [v.strip() for v in val.split(',')]
188 return val
189
191 """
192 Populates self._config with a new ConfigParser instance
193 """
194 cfg = self._get_cfg_fp()
195 try:
196 cp = ConfigParser.ConfigParser()
197 cp.readfp(cfg)
198 self._config = cp
199 try:
200 self.globals = self._load_section('global',
201 self.global_settings)
202 includes = self.globals.get('include')
203 if not includes:
204 return cp
205 mashup = StringIO.StringIO()
206 cfg = self._get_cfg_fp()
207 mashup.write(cfg.read() + '\n')
208 for include in includes:
209 include = os.path.expanduser(include)
210 include = os.path.expandvars(include)
211 try:
212 contents = self._get_cfg_fp(include).read()
213 mashup.write(contents + '\n')
214 except exception.ConfigNotFound:
215 raise exception.ConfigError("include %s not found" %
216 include)
217 mashup.seek(0)
218 cp = ConfigParser.ConfigParser()
219 cp.readfp(mashup)
220 self._config = cp
221 except exception.ConfigSectionMissing:
222 pass
223 return cp
224 except ConfigParser.MissingSectionHeaderError:
225 raise exception.ConfigHasNoSections(cfg.name)
226 except ConfigParser.ParsingError, e:
227 raise exception.ConfigError(e)
228
230 """
231 Reloads the configuration file
232 """
233 self.__load_config()
234 return self.load()
235
236 @property
238 if self._config is None:
239 self._config = self.__load_config()
240 return self._config
241
242 - def _load_settings(self, section_name, settings, store,
243 filter_settings=True):
244 """
245 Load section settings into a dictionary
246 """
247 section = self.config._sections.get(section_name)
248 if not section:
249 raise exception.ConfigSectionMissing(
250 'Missing section %s in config' % section_name)
251 store.update(section)
252 section_conf = store
253 for setting in settings:
254 requirements = settings[setting]
255 func, required, default, options, callback = requirements
256 func = self.type_validators.get(func)
257 value = func(self.config, section_name, setting)
258 if value is not None:
259 if options and not value in options:
260 raise exception.ConfigError(
261 '"%s" setting in section "%s" must be one of: %s' %
262 (setting, section_name,
263 ', '.join([str(o) for o in options])))
264 if callback:
265 value = callback(value)
266 section_conf[setting] = value
267 if filter_settings:
268 for key in store.keys():
269 if key not in settings and key != '__name__':
270 store.pop(key)
271
273 """
274 Check that all required settings were specified in the config.
275 Raises ConfigError otherwise.
276
277 Note that if a setting specified has required=True and
278 default is not None then this method will not raise an error
279 because a default was given. In short, if a setting is required
280 you must provide None as the 'default' value.
281 """
282 section_conf = store
283 for setting in settings:
284 requirements = settings[setting]
285 required = requirements[1]
286 value = section_conf.get(setting)
287 if value is None and required:
288 raise exception.ConfigError(
289 'missing required option %s in section "%s"' %
290 (setting, section_name))
291
293 """
294 Sets the default for each setting in settings regardless of whether
295 the setting was specified in the config or not.
296 """
297 section_conf = store
298 for setting in settings:
299 default = settings[setting][2]
300 if section_conf.get(setting) is None:
301 if DEBUG_CONFIG:
302 log.debug('%s setting not specified. Defaulting to %s' % \
303 (setting, default))
304 section_conf[setting] = default
305
307 """
308 Loads all settings from other template(s) specified by a section's
309 'extends' setting.
310
311 This method walks a dependency tree of sections from bottom up. Each
312 step is a group of settings for a section in the form of a dictionary.
313 A 'master' dictionary is updated with the settings at each step. This
314 causes the next group of settings to override the previous, and so on.
315 The 'section_name' settings are at the top of the dep tree.
316 """
317 section = store[section_name]
318 extends = section.get('extends')
319 if extends is None:
320 return
321 if DEBUG_CONFIG:
322 log.debug('%s extends %s' % (section_name, extends))
323 extensions = [section]
324 while True:
325 extends = section.get('extends', None)
326 if not extends:
327 break
328 try:
329 section = store[extends]
330 if section in extensions:
331 exts = ', '.join([self._get_section_name(x['__name__'])
332 for x in extensions])
333 raise exception.ConfigError(
334 ("Cyclical dependency between sections %s. " % exts) \
335 + "Check your extends settings.")
336 extensions.insert(0, section)
337 except KeyError:
338 raise exception.ConfigError(
339 "%s can't extend non-existent section %s" % \
340 (section_name, extends))
341 transform = AttributeDict()
342 for extension in extensions:
343 transform.update(extension)
344 store[section_name] = transform
345
347 cluster_section = store
348 keyname = cluster_section.get('keyname')
349 if not keyname:
350 return
351 keypair = self.keys.get(keyname)
352 if keypair is None:
353 raise exception.ConfigError(
354 "keypair '%s' not defined in config" % keyname)
355 cluster_section['keyname'] = keyname
356 cluster_section['key_location'] = keypair.get('key_location')
357
371
373 cluster_section = store
374 plugins = cluster_section.get('plugins')
375 if not plugins or isinstance(plugins[0], AttributeDict):
376 return
377 plugs = []
378 cluster_section['plugins'] = plugs
379 for plugin in plugins:
380 if plugin in self.plugins:
381 p = self.plugins.get(plugin)
382 p['__name__'] = p['__name__'].split()[-1]
383 plugs.append(p)
384 else:
385 raise exception.ConfigError(
386 "plugin '%s' not defined in config" % plugin)
387
389 cluster_section = store
390 permissions = cluster_section.get('permissions')
391 if not permissions or isinstance(permissions, AttributeDict):
392 return
393 perms = AttributeDict()
394 cluster_section['permissions'] = perms
395 for perm in permissions:
396 if perm in self.permissions:
397 p = self.permissions.get(perm)
398 p['__name__'] = p['__name__'].split()[-1]
399 perms[perm] = p
400 else:
401 raise exception.ConfigError(
402 "permission '%s' not defined in config" % perm)
403
405 cluster_section = store
406 instance_types = cluster_section.get('node_instance_type')
407 if isinstance(instance_types, basestring):
408 return
409 itypes = []
410 cluster_section['node_instance_types'] = itypes
411 total_num_nodes = 0
412 choices_string = ', '.join(static.INSTANCE_TYPES.keys())
413 try:
414 default_instance_type = instance_types[-1]
415 if not default_instance_type in static.INSTANCE_TYPES:
416 raise exception.ConfigError(
417 ("invalid node_instance_type specified: '%s'\n" +
418 "must be one of: %s") %
419 (default_instance_type, choices_string))
420 except IndexError:
421 default_instance_type = None
422 cluster_section['node_instance_type'] = default_instance_type
423 for type_spec in instance_types[:-1]:
424 type_spec = type_spec.split(':')
425 if len(type_spec) > 3:
426 raise exception.ConfigError(
427 "invalid node_instance_type item specified: %s" % \
428 type_spec)
429 itype = type_spec[0]
430 itype_image = None
431 itype_num = 1
432 if not itype in static.INSTANCE_TYPES:
433 raise exception.ConfigError(
434 ("invalid type specified (%s) in node_instance_type " + \
435 "item: '%s'\nmust be one of: %s") %
436 (itype, type_spec, choices_string))
437 if len(type_spec) == 2:
438 itype, next_var = type_spec
439 try:
440 itype_num = int(next_var)
441 except (TypeError, ValueError):
442 itype_image = next_var
443 elif len(type_spec) == 3:
444 itype, itype_image, itype_num = type_spec
445 try:
446 itype_num = int(itype_num)
447 if itype_num < 1:
448 raise TypeError
449 total_num_nodes += itype_num
450 except (ValueError, TypeError):
451 raise exception.ConfigError(
452 ("number of instances (%s) of type '%s' must " + \
453 "be an integer > 1") % (itype_num, itype))
454 itype_dic = AttributeDict(size=itype_num, image=itype_image,
455 type=itype)
456 itypes.append(itype_dic)
457
458 - def _load_section(self, section_name, section_settings,
459 filter_settings=True):
460 """
461 Returns a dictionary containing all section_settings for a given
462 section_name by first loading the settings in the config, loading
463 the defaults for all settings not specified, and then checking
464 that all required options have been specified
465 """
466 store = AttributeDict()
467 self._load_settings(section_name, section_settings, store,
468 filter_settings)
469 self._load_defaults(section_settings, store)
470 self._check_required(section_name, section_settings, store)
471 return store
472
474 """
475 Returns section name minus prefix
476 e.g.
477 $ print self._get_section('cluster smallcluster')
478 $ smallcluster
479 """
480 return section.split()[1]
481
483 """
484 Returns all sections starting with section_prefix
485 e.g.
486 $ print self._get_sections('cluster')
487 $ ['cluster smallcluster', 'cluster mediumcluster', ..]
488 """
489 return [s for s in self.config.sections() if
490 s.startswith(section_prefix)]
491
492 - def _load_sections(self, section_prefix, section_settings,
493 filter_settings=True):
494 """
495 Loads all sections starting with section_prefix and returns a
496 dictionary containing the name and dictionary of settings for each
497 section.
498 keys --> section name (as returned by self._get_section_name)
499 values --> dictionary of settings for a given section
500
501 e.g.
502 $ print self._load_sections('volumes', self.plugin_settings)
503
504 {'myvol': {'__name__': 'volume myvol',
505 'device': None,
506 'mount_path': '/home',
507 'partition': 1,
508 'volume_id': 'vol-999999'},
509 'myvol2': {'__name__': 'volume myvol2',
510 'device': None,
511 'mount_path': '/myvol2',
512 'partition': 1,
513 'volume_id': 'vol-999999'},
514 """
515 sections = self._get_sections(section_prefix)
516 sections_store = AttributeDict()
517 for sec in sections:
518 name = self._get_section_name(sec)
519 sections_store[name] = self._load_section(sec, section_settings,
520 filter_settings)
521 return sections_store
522
524 """
525 Loads all cluster sections. Similar to _load_sections but also handles
526 populating specified keypair,volume,plugins,permissions,etc settings
527 """
528 clusters = cluster_sections
529 cluster_store = AttributeDict()
530 for cl in clusters:
531 name = self._get_section_name(cl)
532 cluster_store[name] = AttributeDict()
533 self._load_settings(cl, self.cluster_settings, cluster_store[name])
534 for cl in clusters:
535 name = self._get_section_name(cl)
536 self._load_extends_settings(name, cluster_store)
537 self._load_defaults(self.cluster_settings, cluster_store[name])
538 self._load_keypairs(cluster_store[name])
539 self._load_volumes(cluster_store[name])
540 self._load_plugins(cluster_store[name])
541 self._load_permissions(cluster_store[name])
542 self._load_instance_types(cluster_store[name])
543 self._check_required(cl, self.cluster_settings,
544 cluster_store[name])
545 return cluster_store
546
571
573 """
574 Returns AWS credentials defined in the user's shell
575 environment.
576 """
577 awscreds = {}
578 for key in static.AWS_SETTINGS:
579 if key.upper() in os.environ:
580 awscreds[key] = os.environ.get(key.upper())
581 elif key in os.environ:
582 awscreds[key] = os.environ.get(key)
583 return awscreds
584
586 """
587 Returns AWS credentials defined in the configuration
588 file. Defining any of the AWS settings in the environment
589 overrides the configuration file.
590 """
591
592 self.aws.update(self.get_aws_from_environ())
593 return self.aws
594
597
600 """
601 Returns Cluster instance configured with the settings in the
602 config file.
603
604 template_name is the name of a cluster section defined in the config
605
606 tag_name, if specified, will be passed to Cluster instance
607 as cluster_tag
608 """
609 try:
610 kwargs = {}
611 if tag_name:
612 kwargs.update(dict(cluster_tag=tag_name))
613 kwargs.update(self.clusters[template_name])
614 if not ec2_conn:
615 ec2_conn = self.get_easy_ec2()
616 clust = Cluster(ec2_conn, **kwargs)
617 return clust
618 except KeyError:
619 raise exception.ClusterTemplateDoesNotExist(template_name)
620
622 """
623 Returns the default_template specified in the [global] section
624 of the config. Raises NoDefaultTemplateFound if no default cluster
625 template has been specified in the config.
626 """
627 default = self.globals.get('default_template')
628 if not default:
629 raise exception.NoDefaultTemplateFound(
630 options=self.clusters.keys())
631 if not default in self.clusters:
632 raise exception.ClusterTemplateDoesNotExist(default)
633 return default
634
640
646
648 try:
649 return self.keys[keyname]
650 except KeyError:
651 raise exception.KeyNotFound(keyname)
652
654 """
655 Factory for EasyEC2 class that attempts to load AWS credentials from
656 the StarCluster config file. Returns an EasyS3 object if
657 successful.
658 """
659 aws = self.get_aws_credentials()
660 try:
661 s3 = awsutils.EasyS3(**aws)
662 return s3
663 except TypeError:
664 raise exception.ConfigError("no aws credentials found")
665
667 """
668 Factory for EasyEC2 class that attempts to load AWS credentials from
669 the StarCluster config file. Returns an EasyEC2 object if
670 successful.
671 """
672 aws = self.get_aws_credentials()
673 try:
674 ec2 = awsutils.EasyEC2(**aws)
675 return ec2
676 except TypeError:
677 raise exception.ConfigError("no aws credentials found")
678
682
683
684 if __name__ == "__main__":
685 from pprint import pprint
686 cfg = StarClusterConfig().load()
687 pprint(cfg.aws)
688 pprint(cfg.clusters)
689 pprint(cfg.keys)
690 pprint(cfg.vols)
691