Package starcluster :: Module config
[hide private]
[frames] | no frames]

Source Code for Module starcluster.config

  1  #!/usr/bin/env python 
  2  import os 
  3  import urllib 
  4  import StringIO 
  5  import ConfigParser 
  6   
  7  from starcluster import utils 
  8  from starcluster import static 
  9  from starcluster import cluster 
 10  from starcluster import awsutils 
 11  from starcluster import exception 
 12  from starcluster.cluster import Cluster 
 13  from starcluster.utils import AttributeDict 
 14   
 15  from starcluster.logger import log 
 16   
 17  DEBUG_CONFIG = False 
18 19 20 -def get_easy_s3(config_file=None, cache=False):
21 """ 22 Factory for EasyS3 class that attempts to load AWS credentials from 23 the StarCluster config file. Returns an EasyS3 object if 24 successful. 25 """ 26 cfg = get_config(config_file, cache) 27 return cfg.get_easy_s3()
28
29 30 -def get_easy_ec2(config_file=None, cache=False):
31 """ 32 Factory for EasyEC2 class that attempts to load AWS credentials from 33 the StarCluster config file. Returns an EasyEC2 object if 34 successful. 35 """ 36 cfg = get_config(config_file, cache) 37 return cfg.get_easy_ec2()
38
39 40 -def get_cluster_manager(config_file=None, cache=False):
41 """ 42 Factory for ClusterManager class that attempts to load AWS credentials from 43 the StarCluster config file. Returns a ClusterManager object if successful 44 """ 45 cfg = get_config(config_file, cache) 46 return cfg.get_cluster_manager()
47
48 49 -def get_config(config_file=None, cache=False):
50 """Factory for StarClusterConfig object""" 51 return StarClusterConfig(config_file, cache)
52
53 54 -class StarClusterConfig(object):
55 """ 56 Loads StarCluster configuration settings defined in config_file 57 which defaults to ~/.starclustercfg 58 59 Settings are available as follows: 60 61 cfg = StarClusterConfig() 62 or 63 cfg = StarClusterConfig('/path/to/my/config.cfg') 64 cfg.load() 65 aws_info = cfg.aws 66 cluster_cfg = cfg.clusters['mycluster'] 67 key_cfg = cfg.keys['gsg-keypair'] 68 print cluster_cfg 69 """ 70 71 global_settings = static.GLOBAL_SETTINGS 72 aws_settings = static.AWS_SETTINGS 73 key_settings = static.KEY_SETTINGS 74 volume_settings = static.EBS_VOLUME_SETTINGS 75 plugin_settings = static.PLUGIN_SETTINGS 76 cluster_settings = static.CLUSTER_SETTINGS 77 permission_settings = static.PERMISSION_SETTINGS 78 79 # until i can find a way to query AWS for instance types... 80 instance_types = static.INSTANCE_TYPES 81
82 - def __init__(self, config_file=None, cache=False):
83 self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE 84 self.cfg_file = os.path.expanduser(self.cfg_file) 85 self.cfg_file = os.path.expandvars(self.cfg_file) 86 self.type_validators = { 87 int: self._get_int, 88 float: self._get_float, 89 str: self._get_string, 90 bool: self._get_bool, 91 list: self._get_list, 92 } 93 self._config = None 94 self.globals = AttributeDict() 95 self.aws = AttributeDict() 96 self.clusters = AttributeDict() 97 self.keys = AttributeDict() 98 self.vols = AttributeDict() 99 self.plugins = AttributeDict() 100 self.permissions = AttributeDict() 101 self.cache = cache
102
103 - def __repr__(self):
104 return "<StarClusterConfig: %s>" % self.cfg_file
105
106 - def _get_urlfp(self, url):
107 log.debug("Loading url: %s" % url) 108 try: 109 fp = urllib.urlopen(url) 110 if fp.getcode() == 404: 111 raise exception.ConfigError("url %s does not exist" % url) 112 fp.name = url 113 return fp 114 except IOError, e: 115 raise exception.ConfigError( 116 "error loading config from url %s\n%s" % (url, e))
117
118 - def _get_fp(self, cfg_file):
119 log.debug("Loading file: %s" % cfg_file) 120 if os.path.exists(cfg_file): 121 if not os.path.isfile(cfg_file): 122 raise exception.ConfigError( 123 'config %s exists but is not a regular file' % cfg_file) 124 else: 125 raise exception.ConfigNotFound("config file %s does not exist\n" % 126 cfg_file, cfg_file) 127 return open(cfg_file)
128
129 - def _get_cfg_fp(self, cfg_file=None):
130 cfg = cfg_file or self.cfg_file 131 if utils.is_url(cfg): 132 return self._get_urlfp(cfg) 133 else: 134 return self._get_fp(cfg)
135
136 - def _get_bool(self, config, section, option):
137 try: 138 opt = config.getboolean(section, option) 139 return opt 140 except ConfigParser.NoSectionError: 141 pass 142 except ConfigParser.NoOptionError: 143 pass 144 except ValueError: 145 raise exception.ConfigError( 146 "Expected True/False value for setting %s in section [%s]" % 147 (option, section))
148
149 - def _get_int(self, config, section, option):
150 try: 151 opt = config.getint(section, option) 152 return opt 153 except ConfigParser.NoSectionError: 154 pass 155 except ConfigParser.NoOptionError: 156 pass 157 except ValueError: 158 raise exception.ConfigError( 159 "Expected integer value for setting %s in section [%s]" % 160 (option, section))
161
162 - def _get_float(self, config, section, option):
163 try: 164 opt = config.getfloat(section, option) 165 return opt 166 except ConfigParser.NoSectionError: 167 pass 168 except ConfigParser.NoOptionError: 169 pass 170 except ValueError: 171 raise exception.ConfigError( 172 "Expected float value for setting %s in section [%s]" % 173 (option, section))
174
175 - def _get_string(self, config, section, option):
176 try: 177 opt = config.get(section, option) 178 return opt 179 except ConfigParser.NoSectionError: 180 pass 181 except ConfigParser.NoOptionError: 182 pass
183
184 - def _get_list(self, config, section, option):
185 val = self._get_string(config, section, option) 186 if val: 187 val = [v.strip() for v in val.split(',')] 188 return val
189
190 - def __load_config(self):
191 """ 192 Populates self._config with a new ConfigParser instance 193 """ 194 cfg = self._get_cfg_fp() 195 try: 196 cp = ConfigParser.ConfigParser() 197 cp.readfp(cfg) 198 self._config = cp 199 try: 200 self.globals = self._load_section('global', 201 self.global_settings) 202 includes = self.globals.get('include') 203 if not includes: 204 return cp 205 mashup = StringIO.StringIO() 206 cfg = self._get_cfg_fp() 207 mashup.write(cfg.read() + '\n') 208 for include in includes: 209 include = os.path.expanduser(include) 210 include = os.path.expandvars(include) 211 try: 212 contents = self._get_cfg_fp(include).read() 213 mashup.write(contents + '\n') 214 except exception.ConfigNotFound: 215 raise exception.ConfigError("include %s not found" % 216 include) 217 mashup.seek(0) 218 cp = ConfigParser.ConfigParser() 219 cp.readfp(mashup) 220 self._config = cp 221 except exception.ConfigSectionMissing: 222 pass 223 return cp 224 except ConfigParser.MissingSectionHeaderError: 225 raise exception.ConfigHasNoSections(cfg.name) 226 except ConfigParser.ParsingError, e: 227 raise exception.ConfigError(e)
228
229 - def reload(self):
230 """ 231 Reloads the configuration file 232 """ 233 self.__load_config() 234 return self.load()
235 236 @property
237 - def config(self):
238 if self._config is None: 239 self._config = self.__load_config() 240 return self._config
241
242 - def _load_settings(self, section_name, settings, store, 243 filter_settings=True):
244 """ 245 Load section settings into a dictionary 246 """ 247 section = self.config._sections.get(section_name) 248 if not section: 249 raise exception.ConfigSectionMissing( 250 'Missing section %s in config' % section_name) 251 store.update(section) 252 section_conf = store 253 for setting in settings: 254 requirements = settings[setting] 255 func, required, default, options, callback = requirements 256 func = self.type_validators.get(func) 257 value = func(self.config, section_name, setting) 258 if value is not None: 259 if options and not value in options: 260 raise exception.ConfigError( 261 '"%s" setting in section "%s" must be one of: %s' % 262 (setting, section_name, 263 ', '.join([str(o) for o in options]))) 264 if callback: 265 value = callback(value) 266 section_conf[setting] = value 267 if filter_settings: 268 for key in store.keys(): 269 if key not in settings and key != '__name__': 270 store.pop(key)
271
272 - def _check_required(self, section_name, settings, store):
273 """ 274 Check that all required settings were specified in the config. 275 Raises ConfigError otherwise. 276 277 Note that if a setting specified has required=True and 278 default is not None then this method will not raise an error 279 because a default was given. In short, if a setting is required 280 you must provide None as the 'default' value. 281 """ 282 section_conf = store 283 for setting in settings: 284 requirements = settings[setting] 285 required = requirements[1] 286 value = section_conf.get(setting) 287 if value is None and required: 288 raise exception.ConfigError( 289 'missing required option %s in section "%s"' % 290 (setting, section_name))
291
292 - def _load_defaults(self, settings, store):
293 """ 294 Sets the default for each setting in settings regardless of whether 295 the setting was specified in the config or not. 296 """ 297 section_conf = store 298 for setting in settings: 299 default = settings[setting][2] 300 if section_conf.get(setting) is None: 301 if DEBUG_CONFIG: 302 log.debug('%s setting not specified. Defaulting to %s' % \ 303 (setting, default)) 304 section_conf[setting] = default
305
306 - def _load_extends_settings(self, section_name, store):
307 """ 308 Loads all settings from other template(s) specified by a section's 309 'extends' setting. 310 311 This method walks a dependency tree of sections from bottom up. Each 312 step is a group of settings for a section in the form of a dictionary. 313 A 'master' dictionary is updated with the settings at each step. This 314 causes the next group of settings to override the previous, and so on. 315 The 'section_name' settings are at the top of the dep tree. 316 """ 317 section = store[section_name] 318 extends = section.get('extends') 319 if extends is None: 320 return 321 if DEBUG_CONFIG: 322 log.debug('%s extends %s' % (section_name, extends)) 323 extensions = [section] 324 while True: 325 extends = section.get('extends', None) 326 if not extends: 327 break 328 try: 329 section = store[extends] 330 if section in extensions: 331 exts = ', '.join([self._get_section_name(x['__name__']) 332 for x in extensions]) 333 raise exception.ConfigError( 334 ("Cyclical dependency between sections %s. " % exts) \ 335 + "Check your extends settings.") 336 extensions.insert(0, section) 337 except KeyError: 338 raise exception.ConfigError( 339 "%s can't extend non-existent section %s" % \ 340 (section_name, extends)) 341 transform = AttributeDict() 342 for extension in extensions: 343 transform.update(extension) 344 store[section_name] = transform
345
346 - def _load_keypairs(self, store):
347 cluster_section = store 348 keyname = cluster_section.get('keyname') 349 if not keyname: 350 return 351 keypair = self.keys.get(keyname) 352 if keypair is None: 353 raise exception.ConfigError( 354 "keypair '%s' not defined in config" % keyname) 355 cluster_section['keyname'] = keyname 356 cluster_section['key_location'] = keypair.get('key_location')
357
358 - def _load_volumes(self, store):
359 cluster_section = store 360 volumes = cluster_section.get('volumes') 361 if not volumes or isinstance(volumes, AttributeDict): 362 return 363 vols = AttributeDict() 364 cluster_section['volumes'] = vols 365 for volume in volumes: 366 if not volume in self.vols: 367 raise exception.ConfigError( 368 "volume '%s' not defined in config" % volume) 369 vol = self.vols.get(volume) 370 vols[volume] = vol
371
372 - def _load_plugins(self, store):
373 cluster_section = store 374 plugins = cluster_section.get('plugins') 375 if not plugins or isinstance(plugins[0], AttributeDict): 376 return 377 plugs = [] 378 cluster_section['plugins'] = plugs 379 for plugin in plugins: 380 if plugin in self.plugins: 381 p = self.plugins.get(plugin) 382 p['__name__'] = p['__name__'].split()[-1] 383 plugs.append(p) 384 else: 385 raise exception.ConfigError( 386 "plugin '%s' not defined in config" % plugin)
387
388 - def _load_permissions(self, store):
389 cluster_section = store 390 permissions = cluster_section.get('permissions') 391 if not permissions or isinstance(permissions, AttributeDict): 392 return 393 perms = AttributeDict() 394 cluster_section['permissions'] = perms 395 for perm in permissions: 396 if perm in self.permissions: 397 p = self.permissions.get(perm) 398 p['__name__'] = p['__name__'].split()[-1] 399 perms[perm] = p 400 else: 401 raise exception.ConfigError( 402 "permission '%s' not defined in config" % perm)
403
404 - def _load_instance_types(self, store):
405 cluster_section = store 406 instance_types = cluster_section.get('node_instance_type') 407 if isinstance(instance_types, basestring): 408 return 409 itypes = [] 410 cluster_section['node_instance_types'] = itypes 411 total_num_nodes = 0 412 choices_string = ', '.join(static.INSTANCE_TYPES.keys()) 413 try: 414 default_instance_type = instance_types[-1] 415 if not default_instance_type in static.INSTANCE_TYPES: 416 raise exception.ConfigError( 417 ("invalid node_instance_type specified: '%s'\n" + 418 "must be one of: %s") % 419 (default_instance_type, choices_string)) 420 except IndexError: 421 default_instance_type = None 422 cluster_section['node_instance_type'] = default_instance_type 423 for type_spec in instance_types[:-1]: 424 type_spec = type_spec.split(':') 425 if len(type_spec) > 3: 426 raise exception.ConfigError( 427 "invalid node_instance_type item specified: %s" % \ 428 type_spec) 429 itype = type_spec[0] 430 itype_image = None 431 itype_num = 1 432 if not itype in static.INSTANCE_TYPES: 433 raise exception.ConfigError( 434 ("invalid type specified (%s) in node_instance_type " + \ 435 "item: '%s'\nmust be one of: %s") % 436 (itype, type_spec, choices_string)) 437 if len(type_spec) == 2: 438 itype, next_var = type_spec 439 try: 440 itype_num = int(next_var) 441 except (TypeError, ValueError): 442 itype_image = next_var 443 elif len(type_spec) == 3: 444 itype, itype_image, itype_num = type_spec 445 try: 446 itype_num = int(itype_num) 447 if itype_num < 1: 448 raise TypeError 449 total_num_nodes += itype_num 450 except (ValueError, TypeError): 451 raise exception.ConfigError( 452 ("number of instances (%s) of type '%s' must " + \ 453 "be an integer > 1") % (itype_num, itype)) 454 itype_dic = AttributeDict(size=itype_num, image=itype_image, 455 type=itype) 456 itypes.append(itype_dic)
457
458 - def _load_section(self, section_name, section_settings, 459 filter_settings=True):
460 """ 461 Returns a dictionary containing all section_settings for a given 462 section_name by first loading the settings in the config, loading 463 the defaults for all settings not specified, and then checking 464 that all required options have been specified 465 """ 466 store = AttributeDict() 467 self._load_settings(section_name, section_settings, store, 468 filter_settings) 469 self._load_defaults(section_settings, store) 470 self._check_required(section_name, section_settings, store) 471 return store
472
473 - def _get_section_name(self, section):
474 """ 475 Returns section name minus prefix 476 e.g. 477 $ print self._get_section('cluster smallcluster') 478 $ smallcluster 479 """ 480 return section.split()[1]
481
482 - def _get_sections(self, section_prefix):
483 """ 484 Returns all sections starting with section_prefix 485 e.g. 486 $ print self._get_sections('cluster') 487 $ ['cluster smallcluster', 'cluster mediumcluster', ..] 488 """ 489 return [s for s in self.config.sections() if 490 s.startswith(section_prefix)]
491
492 - def _load_sections(self, section_prefix, section_settings, 493 filter_settings=True):
494 """ 495 Loads all sections starting with section_prefix and returns a 496 dictionary containing the name and dictionary of settings for each 497 section. 498 keys --> section name (as returned by self._get_section_name) 499 values --> dictionary of settings for a given section 500 501 e.g. 502 $ print self._load_sections('volumes', self.plugin_settings) 503 504 {'myvol': {'__name__': 'volume myvol', 505 'device': None, 506 'mount_path': '/home', 507 'partition': 1, 508 'volume_id': 'vol-999999'}, 509 'myvol2': {'__name__': 'volume myvol2', 510 'device': None, 511 'mount_path': '/myvol2', 512 'partition': 1, 513 'volume_id': 'vol-999999'}, 514 """ 515 sections = self._get_sections(section_prefix) 516 sections_store = AttributeDict() 517 for sec in sections: 518 name = self._get_section_name(sec) 519 sections_store[name] = self._load_section(sec, section_settings, 520 filter_settings) 521 return sections_store
522
523 - def _load_cluster_sections(self, cluster_sections):
524 """ 525 Loads all cluster sections. Similar to _load_sections but also handles 526 populating specified keypair,volume,plugins,permissions,etc settings 527 """ 528 clusters = cluster_sections 529 cluster_store = AttributeDict() 530 for cl in clusters: 531 name = self._get_section_name(cl) 532 cluster_store[name] = AttributeDict() 533 self._load_settings(cl, self.cluster_settings, cluster_store[name]) 534 for cl in clusters: 535 name = self._get_section_name(cl) 536 self._load_extends_settings(name, cluster_store) 537 self._load_defaults(self.cluster_settings, cluster_store[name]) 538 self._load_keypairs(cluster_store[name]) 539 self._load_volumes(cluster_store[name]) 540 self._load_plugins(cluster_store[name]) 541 self._load_permissions(cluster_store[name]) 542 self._load_instance_types(cluster_store[name]) 543 self._check_required(cl, self.cluster_settings, 544 cluster_store[name]) 545 return cluster_store
546
547 - def load(self):
548 """ 549 Populate this config object from the StarCluster config 550 """ 551 log.debug('Loading config') 552 try: 553 self.globals = self._load_section('global', self.global_settings) 554 except exception.ConfigSectionMissing: 555 pass 556 try: 557 self.aws = self._load_section('aws info', self.aws_settings) 558 except exception.ConfigSectionMissing: 559 log.warn("no [aws info] section found in config") 560 log.warn("attempting to load credentials from environment...") 561 self.aws.update(self.get_aws_from_environ()) 562 self.keys = self._load_sections('key', self.key_settings) 563 self.vols = self._load_sections('volume', self.volume_settings) 564 self.plugins = self._load_sections('plugin', self.plugin_settings, 565 filter_settings=False) 566 self.permissions = self._load_sections('permission', 567 self.permission_settings) 568 sections = self._get_sections('cluster') 569 self.clusters = self._load_cluster_sections(sections) 570 return self
571
572 - def get_aws_from_environ(self):
573 """ 574 Returns AWS credentials defined in the user's shell 575 environment. 576 """ 577 awscreds = {} 578 for key in static.AWS_SETTINGS: 579 if key.upper() in os.environ: 580 awscreds[key] = os.environ.get(key.upper()) 581 elif key in os.environ: 582 awscreds[key] = os.environ.get(key) 583 return awscreds
584
585 - def get_aws_credentials(self):
586 """ 587 Returns AWS credentials defined in the configuration 588 file. Defining any of the AWS settings in the environment 589 overrides the configuration file. 590 """ 591 # first override with environment settings if they exist 592 self.aws.update(self.get_aws_from_environ()) 593 return self.aws
594
595 - def get_cluster_names(self):
596 return self.clusters
597
598 - def get_cluster_template(self, template_name, tag_name=None, 599 ec2_conn=None):
600 """ 601 Returns Cluster instance configured with the settings in the 602 config file. 603 604 template_name is the name of a cluster section defined in the config 605 606 tag_name, if specified, will be passed to Cluster instance 607 as cluster_tag 608 """ 609 try: 610 kwargs = {} 611 if tag_name: 612 kwargs.update(dict(cluster_tag=tag_name)) 613 kwargs.update(self.clusters[template_name]) 614 if not ec2_conn: 615 ec2_conn = self.get_easy_ec2() 616 clust = Cluster(ec2_conn, **kwargs) 617 return clust 618 except KeyError: 619 raise exception.ClusterTemplateDoesNotExist(template_name)
620
622 """ 623 Returns the default_template specified in the [global] section 624 of the config. Raises NoDefaultTemplateFound if no default cluster 625 template has been specified in the config. 626 """ 627 default = self.globals.get('default_template') 628 if not default: 629 raise exception.NoDefaultTemplateFound( 630 options=self.clusters.keys()) 631 if not default in self.clusters: 632 raise exception.ClusterTemplateDoesNotExist(default) 633 return default
634
635 - def get_clusters(self):
636 clusters = [] 637 for cl in self.clusters: 638 cl.append(self.get_cluster_template(cluster)) 639 return clusters
640
641 - def get_plugin(self, plugin):
642 try: 643 return self.plugins[plugin] 644 except KeyError: 645 raise exception.PluginNotFound(plugin)
646
647 - def get_key(self, keyname):
648 try: 649 return self.keys[keyname] 650 except KeyError: 651 raise exception.KeyNotFound(keyname)
652
653 - def get_easy_s3(self):
654 """ 655 Factory for EasyEC2 class that attempts to load AWS credentials from 656 the StarCluster config file. Returns an EasyS3 object if 657 successful. 658 """ 659 aws = self.get_aws_credentials() 660 try: 661 s3 = awsutils.EasyS3(**aws) 662 return s3 663 except TypeError: 664 raise exception.ConfigError("no aws credentials found")
665
666 - def get_easy_ec2(self):
667 """ 668 Factory for EasyEC2 class that attempts to load AWS credentials from 669 the StarCluster config file. Returns an EasyEC2 object if 670 successful. 671 """ 672 aws = self.get_aws_credentials() 673 try: 674 ec2 = awsutils.EasyEC2(**aws) 675 return ec2 676 except TypeError: 677 raise exception.ConfigError("no aws credentials found")
678
679 - def get_cluster_manager(self):
680 ec2 = self.get_easy_ec2() 681 return cluster.ClusterManager(self, ec2)
682 683 684 if __name__ == "__main__": 685 from pprint import pprint 686 cfg = StarClusterConfig().load() 687 pprint(cfg.aws) 688 pprint(cfg.clusters) 689 pprint(cfg.keys) 690 pprint(cfg.vols) 691