1 import os
2 import urllib
3 import StringIO
4 import ConfigParser
5
6 from starcluster import utils
7 from starcluster import static
8 from starcluster import cluster
9 from starcluster import awsutils
10 from starcluster import exception
11 from starcluster.cluster import Cluster
12 from starcluster.utils import AttributeDict
13
14 from starcluster.logger import log
15
16 DEBUG_CONFIG = False
20 """
21 Factory for EasyS3 class that attempts to load AWS credentials from
22 the StarCluster config file. Returns an EasyS3 object if
23 successful.
24 """
25 cfg = get_config(config_file, cache)
26 return cfg.get_easy_s3()
27
30 """
31 Factory for EasyEC2 class that attempts to load AWS credentials from
32 the StarCluster config file. Returns an EasyEC2 object if
33 successful.
34 """
35 cfg = get_config(config_file, cache)
36 return cfg.get_easy_ec2()
37
40 """
41 Factory for ClusterManager class that attempts to load AWS credentials from
42 the StarCluster config file. Returns a ClusterManager object if successful
43 """
44 cfg = get_config(config_file, cache)
45 return cfg.get_cluster_manager()
46
49 """Factory for StarClusterConfig object"""
50 return StarClusterConfig(config_file, cache)
51
54 """
55 Loads StarCluster configuration settings defined in config_file
56 which defaults to ~/.starclustercfg
57
58 Settings are available as follows:
59
60 cfg = StarClusterConfig()
61 or
62 cfg = StarClusterConfig('/path/to/my/config.cfg')
63 cfg.load()
64 aws_info = cfg.aws
65 cluster_cfg = cfg.clusters['mycluster']
66 key_cfg = cfg.keys['gsg-keypair']
67 print cluster_cfg
68 """
69
70 global_settings = static.GLOBAL_SETTINGS
71 aws_settings = static.AWS_SETTINGS
72 key_settings = static.KEY_SETTINGS
73 volume_settings = static.EBS_VOLUME_SETTINGS
74 plugin_settings = static.PLUGIN_SETTINGS
75 cluster_settings = static.CLUSTER_SETTINGS
76 permission_settings = static.PERMISSION_SETTINGS
77
78
79 instance_types = static.INSTANCE_TYPES
80
81 - def __init__(self, config_file=None, cache=False):
101
103 return "<StarClusterConfig: %s>" % self.cfg_file
104
106 log.debug("Loading url: %s" % url)
107 try:
108 fp = urllib.urlopen(url)
109 if fp.getcode() == 404:
110 raise exception.ConfigError("url %s does not exist" % url)
111 fp.name = url
112 return fp
113 except IOError, e:
114 raise exception.ConfigError(
115 "error loading config from url %s\n%s" % (url, e))
116
118 log.debug("Loading file: %s" % cfg_file)
119 if os.path.exists(cfg_file):
120 if not os.path.isfile(cfg_file):
121 raise exception.ConfigError(
122 'config %s exists but is not a regular file' % cfg_file)
123 else:
124 raise exception.ConfigNotFound("config file %s does not exist\n" %
125 cfg_file, cfg_file)
126 return open(cfg_file)
127
134
135 - def _get_bool(self, config, section, option):
136 try:
137 opt = config.getboolean(section, option)
138 return opt
139 except ConfigParser.NoSectionError:
140 pass
141 except ConfigParser.NoOptionError:
142 pass
143 except ValueError:
144 raise exception.ConfigError(
145 "Expected True/False value for setting %s in section [%s]" %
146 (option, section))
147
148 - def _get_int(self, config, section, option):
149 try:
150 opt = config.getint(section, option)
151 return opt
152 except ConfigParser.NoSectionError:
153 pass
154 except ConfigParser.NoOptionError:
155 pass
156 except ValueError:
157 raise exception.ConfigError(
158 "Expected integer value for setting %s in section [%s]" %
159 (option, section))
160
162 try:
163 opt = config.getfloat(section, option)
164 return opt
165 except ConfigParser.NoSectionError:
166 pass
167 except ConfigParser.NoOptionError:
168 pass
169 except ValueError:
170 raise exception.ConfigError(
171 "Expected float value for setting %s in section [%s]" %
172 (option, section))
173
175 try:
176 opt = config.get(section, option)
177 return opt
178 except ConfigParser.NoSectionError:
179 pass
180 except ConfigParser.NoOptionError:
181 pass
182
183 - def _get_list(self, config, section, option):
184 val = self._get_string(config, section, option)
185 if val:
186 val = [v.strip() for v in val.split(',')]
187 return val
188
190 """
191 Populates self._config with a new ConfigParser instance
192 """
193 cfg = self._get_cfg_fp()
194 try:
195 cp = ConfigParser.ConfigParser()
196 cp.readfp(cfg)
197 self._config = cp
198 try:
199 self.globals = self._load_section('global',
200 self.global_settings)
201 includes = self.globals.get('include')
202 if not includes:
203 return cp
204 mashup = StringIO.StringIO()
205 cfg = self._get_cfg_fp()
206 mashup.write(cfg.read() + '\n')
207 for include in includes:
208 include = os.path.expanduser(include)
209 include = os.path.expandvars(include)
210 try:
211 contents = self._get_cfg_fp(include).read()
212 mashup.write(contents + '\n')
213 except exception.ConfigNotFound:
214 raise exception.ConfigError("include %s not found" %
215 include)
216 mashup.seek(0)
217 cp = ConfigParser.ConfigParser()
218 cp.readfp(mashup)
219 self._config = cp
220 except exception.ConfigSectionMissing:
221 pass
222 return cp
223 except ConfigParser.MissingSectionHeaderError:
224 raise exception.ConfigHasNoSections(cfg.name)
225 except ConfigParser.ParsingError, e:
226 raise exception.ConfigError(e)
227
229 """
230 Reloads the configuration file
231 """
232 self.__load_config()
233 return self.load()
234
235 @property
237 if self._config is None:
238 self._config = self.__load_config()
239 return self._config
240
241 - def _load_settings(self, section_name, settings, store,
242 filter_settings=True):
243 """
244 Load section settings into a dictionary
245 """
246 section = self.config._sections.get(section_name)
247 if not section:
248 raise exception.ConfigSectionMissing(
249 'Missing section %s in config' % section_name)
250 store.update(section)
251 section_conf = store
252 for setting in settings:
253 requirements = settings[setting]
254 func, required, default, options, callback = requirements
255 func = self.type_validators.get(func)
256 value = func(self.config, section_name, setting)
257 if value is not None:
258 if options and not value in options:
259 raise exception.ConfigError(
260 '"%s" setting in section "%s" must be one of: %s' %
261 (setting, section_name,
262 ', '.join([str(o) for o in options])))
263 if callback:
264 value = callback(value)
265 section_conf[setting] = value
266 if filter_settings:
267 for key in store.keys():
268 if key not in settings and key != '__name__':
269 store.pop(key)
270
272 """
273 Check that all required settings were specified in the config.
274 Raises ConfigError otherwise.
275
276 Note that if a setting specified has required=True and
277 default is not None then this method will not raise an error
278 because a default was given. In short, if a setting is required
279 you must provide None as the 'default' value.
280 """
281 section_conf = store
282 for setting in settings:
283 requirements = settings[setting]
284 required = requirements[1]
285 value = section_conf.get(setting)
286 if value is None and required:
287 raise exception.ConfigError(
288 'missing required option %s in section "%s"' %
289 (setting, section_name))
290
292 """
293 Sets the default for each setting in settings regardless of whether
294 the setting was specified in the config or not.
295 """
296 section_conf = store
297 for setting in settings:
298 default = settings[setting][2]
299 if section_conf.get(setting) is None:
300 if DEBUG_CONFIG:
301 log.debug('%s setting not specified. Defaulting to %s' %
302 (setting, default))
303 section_conf[setting] = default
304
306 """
307 Loads all settings from other template(s) specified by a section's
308 'extends' setting.
309
310 This method walks a dependency tree of sections from bottom up. Each
311 step is a group of settings for a section in the form of a dictionary.
312 A 'master' dictionary is updated with the settings at each step. This
313 causes the next group of settings to override the previous, and so on.
314 The 'section_name' settings are at the top of the dependency tree.
315 """
316 section = store[section_name]
317 extends = section.get('extends')
318 if extends is None:
319 return
320 if DEBUG_CONFIG:
321 log.debug('%s extends %s' % (section_name, extends))
322 extensions = [section]
323 while True:
324 extends = section.get('extends', None)
325 if not extends:
326 break
327 try:
328 section = store[extends]
329 if section in extensions:
330 exts = ', '.join([self._get_section_name(x['__name__'])
331 for x in extensions])
332 raise exception.ConfigError(
333 "Cyclical dependency between sections %s. "
334 "Check your EXTENDS settings." % exts)
335 extensions.insert(0, section)
336 except KeyError:
337 raise exception.ConfigError(
338 "%s can't extend non-existent section %s" %
339 (section_name, extends))
340 transform = AttributeDict()
341 for extension in extensions:
342 transform.update(extension)
343 store[section_name] = transform
344
346 cluster_section = store
347 keyname = cluster_section.get('keyname')
348 if not keyname:
349 return
350 keypair = self.keys.get(keyname)
351 if keypair is None:
352 raise exception.ConfigError(
353 "keypair '%s' not defined in config" % keyname)
354 cluster_section['keyname'] = keyname
355 cluster_section['key_location'] = keypair.get('key_location')
356
370
372 cluster_section = store
373 plugins = cluster_section.get('plugins')
374 if not plugins or isinstance(plugins[0], AttributeDict):
375 return
376 plugs = []
377 cluster_section['plugins'] = plugs
378 for plugin in plugins:
379 if plugin in self.plugins:
380 p = self.plugins.get(plugin)
381 p['__name__'] = p['__name__'].split()[-1]
382 plugs.append(p)
383 else:
384 raise exception.ConfigError(
385 "plugin '%s' not defined in config" % plugin)
386
388 cluster_section = store
389 permissions = cluster_section.get('permissions')
390 if not permissions or isinstance(permissions, AttributeDict):
391 return
392 perms = AttributeDict()
393 cluster_section['permissions'] = perms
394 for perm in permissions:
395 if perm in self.permissions:
396 p = self.permissions.get(perm)
397 p['__name__'] = p['__name__'].split()[-1]
398 perms[perm] = p
399 else:
400 raise exception.ConfigError(
401 "permission '%s' not defined in config" % perm)
402
404 cluster_section = store
405 instance_types = cluster_section.get('node_instance_type')
406 if isinstance(instance_types, basestring):
407 return
408 itypes = []
409 cluster_section['node_instance_types'] = itypes
410 total_num_nodes = 0
411 choices_string = ', '.join(static.INSTANCE_TYPES.keys())
412 try:
413 default_instance_type = instance_types[-1]
414 if not default_instance_type in static.INSTANCE_TYPES:
415 raise exception.ConfigError(
416 "invalid node_instance_type specified: '%s'\n"
417 "must be one of: %s" %
418 (default_instance_type, choices_string))
419 except IndexError:
420 default_instance_type = None
421 cluster_section['node_instance_type'] = default_instance_type
422 for type_spec in instance_types[:-1]:
423 type_spec = type_spec.split(':')
424 if len(type_spec) > 3:
425 raise exception.ConfigError(
426 "invalid node_instance_type item specified: %s" %
427 type_spec)
428 itype = type_spec[0]
429 itype_image = None
430 itype_num = 1
431 if not itype in static.INSTANCE_TYPES:
432 raise exception.ConfigError(
433 "invalid type specified (%s) in node_instance_type "
434 "item: '%s'\nmust be one of: %s" %
435 (itype, type_spec, choices_string))
436 if len(type_spec) == 2:
437 itype, next_var = type_spec
438 try:
439 itype_num = int(next_var)
440 except (TypeError, ValueError):
441 itype_image = next_var
442 elif len(type_spec) == 3:
443 itype, itype_image, itype_num = type_spec
444 try:
445 itype_num = int(itype_num)
446 if itype_num < 1:
447 raise TypeError
448 total_num_nodes += itype_num
449 except (ValueError, TypeError):
450 raise exception.ConfigError(
451 "number of instances (%s) of type '%s' must "
452 "be an integer > 1" % (itype_num, itype))
453 itype_dic = AttributeDict(size=itype_num, image=itype_image,
454 type=itype)
455 itypes.append(itype_dic)
456
457 - def _load_section(self, section_name, section_settings,
458 filter_settings=True):
459 """
460 Returns a dictionary containing all section_settings for a given
461 section_name by first loading the settings in the config, loading
462 the defaults for all settings not specified, and then checking
463 that all required options have been specified
464 """
465 store = AttributeDict()
466 self._load_settings(section_name, section_settings, store,
467 filter_settings)
468 self._load_defaults(section_settings, store)
469 self._check_required(section_name, section_settings, store)
470 return store
471
473 """
474 Returns section name minus prefix
475 e.g.
476 $ print self._get_section('cluster smallcluster')
477 $ smallcluster
478 """
479 return section.split()[1]
480
482 """
483 Returns all sections starting with section_prefix
484 e.g.
485 $ print self._get_sections('cluster')
486 $ ['cluster smallcluster', 'cluster mediumcluster', ..]
487 """
488 return [s for s in self.config.sections() if
489 s.startswith(section_prefix)]
490
491 - def _load_sections(self, section_prefix, section_settings,
492 filter_settings=True):
493 """
494 Loads all sections starting with section_prefix and returns a
495 dictionary containing the name and dictionary of settings for each
496 section.
497 keys --> section name (as returned by self._get_section_name)
498 values --> dictionary of settings for a given section
499
500 e.g.
501 $ print self._load_sections('volumes', self.plugin_settings)
502
503 {'myvol': {'__name__': 'volume myvol',
504 'device': None,
505 'mount_path': '/home',
506 'partition': 1,
507 'volume_id': 'vol-999999'},
508 'myvol2': {'__name__': 'volume myvol2',
509 'device': None,
510 'mount_path': '/myvol2',
511 'partition': 1,
512 'volume_id': 'vol-999999'},
513 """
514 sections = self._get_sections(section_prefix)
515 sections_store = AttributeDict()
516 for sec in sections:
517 name = self._get_section_name(sec)
518 sections_store[name] = self._load_section(sec, section_settings,
519 filter_settings)
520 return sections_store
521
523 """
524 Loads all cluster sections. Similar to _load_sections but also handles
525 populating specified keypair, volume, plugins, permissions, etc.
526 settings
527 """
528 clusters = cluster_sections
529 cluster_store = AttributeDict()
530 for cl in clusters:
531 name = self._get_section_name(cl)
532 cluster_store[name] = AttributeDict()
533 self._load_settings(cl, self.cluster_settings, cluster_store[name])
534 for cl in clusters:
535 name = self._get_section_name(cl)
536 self._load_extends_settings(name, cluster_store)
537 self._load_defaults(self.cluster_settings, cluster_store[name])
538 self._load_keypairs(cluster_store[name])
539 self._load_volumes(cluster_store[name])
540 self._load_plugins(cluster_store[name])
541 self._load_permissions(cluster_store[name])
542 self._load_instance_types(cluster_store[name])
543 self._check_required(cl, self.cluster_settings,
544 cluster_store[name])
545 return cluster_store
546
571
573 """
574 Returns AWS credentials defined in the user's shell
575 environment.
576 """
577 awscreds = {}
578 for key in static.AWS_SETTINGS:
579 if key.upper() in os.environ:
580 awscreds[key] = os.environ.get(key.upper())
581 elif key in os.environ:
582 awscreds[key] = os.environ.get(key)
583 return awscreds
584
586 """
587 Returns AWS credentials defined in the configuration
588 file. Defining any of the AWS settings in the environment
589 overrides the configuration file.
590 """
591
592 self.aws.update(self.get_aws_from_environ())
593 return self.aws
594
597
600 """
601 Returns Cluster instance configured with the settings in the
602 config file.
603
604 template_name is the name of a cluster section defined in the config
605
606 tag_name, if specified, will be passed to Cluster instance
607 as cluster_tag
608 """
609 try:
610 kwargs = {}
611 if tag_name:
612 kwargs.update(dict(cluster_tag=tag_name))
613 kwargs.update(self.clusters[template_name])
614 if not ec2_conn:
615 ec2_conn = self.get_easy_ec2()
616 clust = Cluster(ec2_conn, **kwargs)
617 return clust
618 except KeyError:
619 raise exception.ClusterTemplateDoesNotExist(template_name)
620
622 """
623 Returns the default_template specified in the [global] section
624 of the config. Raises NoDefaultTemplateFound if no default cluster
625 template has been specified in the config.
626 """
627 default = self.globals.get('default_template')
628 if not default:
629 raise exception.NoDefaultTemplateFound(
630 options=self.clusters.keys())
631 if not default in self.clusters:
632 raise exception.ClusterTemplateDoesNotExist(default)
633 return default
634
640
646
648 try:
649 return self.keys[keyname]
650 except KeyError:
651 raise exception.KeyNotFound(keyname)
652
654 """
655 Factory for EasyEC2 class that attempts to load AWS credentials from
656 the StarCluster config file. Returns an EasyS3 object if
657 successful.
658 """
659 aws = self.get_aws_credentials()
660 try:
661 s3 = awsutils.EasyS3(**aws)
662 return s3
663 except TypeError:
664 raise exception.ConfigError("no aws credentials found")
665
667 """
668 Factory for EasyEC2 class that attempts to load AWS credentials from
669 the StarCluster config file. Returns an EasyEC2 object if
670 successful.
671 """
672 aws = self.get_aws_credentials()
673 try:
674 ec2 = awsutils.EasyEC2(**aws)
675 return ec2
676 except TypeError:
677 raise exception.ConfigError("no aws credentials found")
678
682
683
684 if __name__ == "__main__":
685 from pprint import pprint
686 cfg = StarClusterConfig().load()
687 pprint(cfg.aws)
688 pprint(cfg.clusters)
689 pprint(cfg.keys)
690 pprint(cfg.vols)
691