Package starcluster :: Module config
[hide private]
[frames] | no frames]

Source Code for Module starcluster.config

  1  import os 
  2  import urllib 
  3  import StringIO 
  4  import ConfigParser 
  5   
  6  from starcluster import utils 
  7  from starcluster import static 
  8  from starcluster import cluster 
  9  from starcluster import awsutils 
 10  from starcluster import exception 
 11  from starcluster.cluster import Cluster 
 12  from starcluster.utils import AttributeDict 
 13   
 14  from starcluster.logger import log 
 15   
 16  DEBUG_CONFIG = False 
17 18 19 -def get_easy_s3(config_file=None, cache=False):
20 """ 21 Factory for EasyS3 class that attempts to load AWS credentials from 22 the StarCluster config file. Returns an EasyS3 object if 23 successful. 24 """ 25 cfg = get_config(config_file, cache) 26 return cfg.get_easy_s3()
27
28 29 -def get_easy_ec2(config_file=None, cache=False):
30 """ 31 Factory for EasyEC2 class that attempts to load AWS credentials from 32 the StarCluster config file. Returns an EasyEC2 object if 33 successful. 34 """ 35 cfg = get_config(config_file, cache) 36 return cfg.get_easy_ec2()
37
38 39 -def get_cluster_manager(config_file=None, cache=False):
40 """ 41 Factory for ClusterManager class that attempts to load AWS credentials from 42 the StarCluster config file. Returns a ClusterManager object if successful 43 """ 44 cfg = get_config(config_file, cache) 45 return cfg.get_cluster_manager()
46
47 48 -def get_config(config_file=None, cache=False):
49 """Factory for StarClusterConfig object""" 50 return StarClusterConfig(config_file, cache)
51
52 53 -class StarClusterConfig(object):
54 """ 55 Loads StarCluster configuration settings defined in config_file 56 which defaults to ~/.starclustercfg 57 58 Settings are available as follows: 59 60 cfg = StarClusterConfig() 61 or 62 cfg = StarClusterConfig('/path/to/my/config.cfg') 63 cfg.load() 64 aws_info = cfg.aws 65 cluster_cfg = cfg.clusters['mycluster'] 66 key_cfg = cfg.keys['gsg-keypair'] 67 print cluster_cfg 68 """ 69 70 global_settings = static.GLOBAL_SETTINGS 71 aws_settings = static.AWS_SETTINGS 72 key_settings = static.KEY_SETTINGS 73 volume_settings = static.EBS_VOLUME_SETTINGS 74 plugin_settings = static.PLUGIN_SETTINGS 75 cluster_settings = static.CLUSTER_SETTINGS 76 permission_settings = static.PERMISSION_SETTINGS 77 78 # until i can find a way to query AWS for instance types... 79 instance_types = static.INSTANCE_TYPES 80
81 - def __init__(self, config_file=None, cache=False):
82 self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE 83 self.cfg_file = os.path.expanduser(self.cfg_file) 84 self.cfg_file = os.path.expandvars(self.cfg_file) 85 self.type_validators = { 86 int: self._get_int, 87 float: self._get_float, 88 str: self._get_string, 89 bool: self._get_bool, 90 list: self._get_list, 91 } 92 self._config = None 93 self.globals = AttributeDict() 94 self.aws = AttributeDict() 95 self.clusters = AttributeDict() 96 self.keys = AttributeDict() 97 self.vols = AttributeDict() 98 self.plugins = AttributeDict() 99 self.permissions = AttributeDict() 100 self.cache = cache
101
102 - def __repr__(self):
103 return "<StarClusterConfig: %s>" % self.cfg_file
104
105 - def _get_urlfp(self, url):
106 log.debug("Loading url: %s" % url) 107 try: 108 fp = urllib.urlopen(url) 109 if fp.getcode() == 404: 110 raise exception.ConfigError("url %s does not exist" % url) 111 fp.name = url 112 return fp 113 except IOError, e: 114 raise exception.ConfigError( 115 "error loading config from url %s\n%s" % (url, e))
116
117 - def _get_fp(self, cfg_file):
118 log.debug("Loading file: %s" % cfg_file) 119 if os.path.exists(cfg_file): 120 if not os.path.isfile(cfg_file): 121 raise exception.ConfigError( 122 'config %s exists but is not a regular file' % cfg_file) 123 else: 124 raise exception.ConfigNotFound("config file %s does not exist\n" % 125 cfg_file, cfg_file) 126 return open(cfg_file)
127
128 - def _get_cfg_fp(self, cfg_file=None):
129 cfg = cfg_file or self.cfg_file 130 if utils.is_url(cfg): 131 return self._get_urlfp(cfg) 132 else: 133 return self._get_fp(cfg)
134
135 - def _get_bool(self, config, section, option):
136 try: 137 opt = config.getboolean(section, option) 138 return opt 139 except ConfigParser.NoSectionError: 140 pass 141 except ConfigParser.NoOptionError: 142 pass 143 except ValueError: 144 raise exception.ConfigError( 145 "Expected True/False value for setting %s in section [%s]" % 146 (option, section))
147
148 - def _get_int(self, config, section, option):
149 try: 150 opt = config.getint(section, option) 151 return opt 152 except ConfigParser.NoSectionError: 153 pass 154 except ConfigParser.NoOptionError: 155 pass 156 except ValueError: 157 raise exception.ConfigError( 158 "Expected integer value for setting %s in section [%s]" % 159 (option, section))
160
161 - def _get_float(self, config, section, option):
162 try: 163 opt = config.getfloat(section, option) 164 return opt 165 except ConfigParser.NoSectionError: 166 pass 167 except ConfigParser.NoOptionError: 168 pass 169 except ValueError: 170 raise exception.ConfigError( 171 "Expected float value for setting %s in section [%s]" % 172 (option, section))
173
174 - def _get_string(self, config, section, option):
175 try: 176 opt = config.get(section, option) 177 return opt 178 except ConfigParser.NoSectionError: 179 pass 180 except ConfigParser.NoOptionError: 181 pass
182
183 - def _get_list(self, config, section, option):
184 val = self._get_string(config, section, option) 185 if val: 186 val = [v.strip() for v in val.split(',')] 187 return val
188
189 - def __load_config(self):
190 """ 191 Populates self._config with a new ConfigParser instance 192 """ 193 cfg = self._get_cfg_fp() 194 try: 195 cp = ConfigParser.ConfigParser() 196 cp.readfp(cfg) 197 self._config = cp 198 try: 199 self.globals = self._load_section('global', 200 self.global_settings) 201 includes = self.globals.get('include') 202 if not includes: 203 return cp 204 mashup = StringIO.StringIO() 205 cfg = self._get_cfg_fp() 206 mashup.write(cfg.read() + '\n') 207 for include in includes: 208 include = os.path.expanduser(include) 209 include = os.path.expandvars(include) 210 try: 211 contents = self._get_cfg_fp(include).read() 212 mashup.write(contents + '\n') 213 except exception.ConfigNotFound: 214 raise exception.ConfigError("include %s not found" % 215 include) 216 mashup.seek(0) 217 cp = ConfigParser.ConfigParser() 218 cp.readfp(mashup) 219 self._config = cp 220 except exception.ConfigSectionMissing: 221 pass 222 return cp 223 except ConfigParser.MissingSectionHeaderError: 224 raise exception.ConfigHasNoSections(cfg.name) 225 except ConfigParser.ParsingError, e: 226 raise exception.ConfigError(e)
227
228 - def reload(self):
229 """ 230 Reloads the configuration file 231 """ 232 self.__load_config() 233 return self.load()
234 235 @property
236 - def config(self):
237 if self._config is None: 238 self._config = self.__load_config() 239 return self._config
240
241 - def _load_settings(self, section_name, settings, store, 242 filter_settings=True):
243 """ 244 Load section settings into a dictionary 245 """ 246 section = self.config._sections.get(section_name) 247 if not section: 248 raise exception.ConfigSectionMissing( 249 'Missing section %s in config' % section_name) 250 store.update(section) 251 section_conf = store 252 for setting in settings: 253 requirements = settings[setting] 254 func, required, default, options, callback = requirements 255 func = self.type_validators.get(func) 256 value = func(self.config, section_name, setting) 257 if value is not None: 258 if options and not value in options: 259 raise exception.ConfigError( 260 '"%s" setting in section "%s" must be one of: %s' % 261 (setting, section_name, 262 ', '.join([str(o) for o in options]))) 263 if callback: 264 value = callback(value) 265 section_conf[setting] = value 266 if filter_settings: 267 for key in store.keys(): 268 if key not in settings and key != '__name__': 269 store.pop(key)
270
271 - def _check_required(self, section_name, settings, store):
272 """ 273 Check that all required settings were specified in the config. 274 Raises ConfigError otherwise. 275 276 Note that if a setting specified has required=True and 277 default is not None then this method will not raise an error 278 because a default was given. In short, if a setting is required 279 you must provide None as the 'default' value. 280 """ 281 section_conf = store 282 for setting in settings: 283 requirements = settings[setting] 284 required = requirements[1] 285 value = section_conf.get(setting) 286 if value is None and required: 287 raise exception.ConfigError( 288 'missing required option %s in section "%s"' % 289 (setting, section_name))
290
291 - def _load_defaults(self, settings, store):
292 """ 293 Sets the default for each setting in settings regardless of whether 294 the setting was specified in the config or not. 295 """ 296 section_conf = store 297 for setting in settings: 298 default = settings[setting][2] 299 if section_conf.get(setting) is None: 300 if DEBUG_CONFIG: 301 log.debug('%s setting not specified. Defaulting to %s' % 302 (setting, default)) 303 section_conf[setting] = default
304
305 - def _load_extends_settings(self, section_name, store):
306 """ 307 Loads all settings from other template(s) specified by a section's 308 'extends' setting. 309 310 This method walks a dependency tree of sections from bottom up. Each 311 step is a group of settings for a section in the form of a dictionary. 312 A 'master' dictionary is updated with the settings at each step. This 313 causes the next group of settings to override the previous, and so on. 314 The 'section_name' settings are at the top of the dependency tree. 315 """ 316 section = store[section_name] 317 extends = section.get('extends') 318 if extends is None: 319 return 320 if DEBUG_CONFIG: 321 log.debug('%s extends %s' % (section_name, extends)) 322 extensions = [section] 323 while True: 324 extends = section.get('extends', None) 325 if not extends: 326 break 327 try: 328 section = store[extends] 329 if section in extensions: 330 exts = ', '.join([self._get_section_name(x['__name__']) 331 for x in extensions]) 332 raise exception.ConfigError( 333 "Cyclical dependency between sections %s. " 334 "Check your EXTENDS settings." % exts) 335 extensions.insert(0, section) 336 except KeyError: 337 raise exception.ConfigError( 338 "%s can't extend non-existent section %s" % 339 (section_name, extends)) 340 transform = AttributeDict() 341 for extension in extensions: 342 transform.update(extension) 343 store[section_name] = transform
344
345 - def _load_keypairs(self, store):
346 cluster_section = store 347 keyname = cluster_section.get('keyname') 348 if not keyname: 349 return 350 keypair = self.keys.get(keyname) 351 if keypair is None: 352 raise exception.ConfigError( 353 "keypair '%s' not defined in config" % keyname) 354 cluster_section['keyname'] = keyname 355 cluster_section['key_location'] = keypair.get('key_location')
356
357 - def _load_volumes(self, store):
358 cluster_section = store 359 volumes = cluster_section.get('volumes') 360 if not volumes or isinstance(volumes, AttributeDict): 361 return 362 vols = AttributeDict() 363 cluster_section['volumes'] = vols 364 for volume in volumes: 365 if not volume in self.vols: 366 raise exception.ConfigError( 367 "volume '%s' not defined in config" % volume) 368 vol = self.vols.get(volume) 369 vols[volume] = vol
370
371 - def _load_plugins(self, store):
372 cluster_section = store 373 plugins = cluster_section.get('plugins') 374 if not plugins or isinstance(plugins[0], AttributeDict): 375 return 376 plugs = [] 377 cluster_section['plugins'] = plugs 378 for plugin in plugins: 379 if plugin in self.plugins: 380 p = self.plugins.get(plugin) 381 p['__name__'] = p['__name__'].split()[-1] 382 plugs.append(p) 383 else: 384 raise exception.ConfigError( 385 "plugin '%s' not defined in config" % plugin)
386
387 - def _load_permissions(self, store):
388 cluster_section = store 389 permissions = cluster_section.get('permissions') 390 if not permissions or isinstance(permissions, AttributeDict): 391 return 392 perms = AttributeDict() 393 cluster_section['permissions'] = perms 394 for perm in permissions: 395 if perm in self.permissions: 396 p = self.permissions.get(perm) 397 p['__name__'] = p['__name__'].split()[-1] 398 perms[perm] = p 399 else: 400 raise exception.ConfigError( 401 "permission '%s' not defined in config" % perm)
402
403 - def _load_instance_types(self, store):
404 cluster_section = store 405 instance_types = cluster_section.get('node_instance_type') 406 if isinstance(instance_types, basestring): 407 return 408 itypes = [] 409 cluster_section['node_instance_types'] = itypes 410 total_num_nodes = 0 411 choices_string = ', '.join(static.INSTANCE_TYPES.keys()) 412 try: 413 default_instance_type = instance_types[-1] 414 if not default_instance_type in static.INSTANCE_TYPES: 415 raise exception.ConfigError( 416 "invalid node_instance_type specified: '%s'\n" 417 "must be one of: %s" % 418 (default_instance_type, choices_string)) 419 except IndexError: 420 default_instance_type = None 421 cluster_section['node_instance_type'] = default_instance_type 422 for type_spec in instance_types[:-1]: 423 type_spec = type_spec.split(':') 424 if len(type_spec) > 3: 425 raise exception.ConfigError( 426 "invalid node_instance_type item specified: %s" % 427 type_spec) 428 itype = type_spec[0] 429 itype_image = None 430 itype_num = 1 431 if not itype in static.INSTANCE_TYPES: 432 raise exception.ConfigError( 433 "invalid type specified (%s) in node_instance_type " 434 "item: '%s'\nmust be one of: %s" % 435 (itype, type_spec, choices_string)) 436 if len(type_spec) == 2: 437 itype, next_var = type_spec 438 try: 439 itype_num = int(next_var) 440 except (TypeError, ValueError): 441 itype_image = next_var 442 elif len(type_spec) == 3: 443 itype, itype_image, itype_num = type_spec 444 try: 445 itype_num = int(itype_num) 446 if itype_num < 1: 447 raise TypeError 448 total_num_nodes += itype_num 449 except (ValueError, TypeError): 450 raise exception.ConfigError( 451 "number of instances (%s) of type '%s' must " 452 "be an integer > 1" % (itype_num, itype)) 453 itype_dic = AttributeDict(size=itype_num, image=itype_image, 454 type=itype) 455 itypes.append(itype_dic)
456
457 - def _load_section(self, section_name, section_settings, 458 filter_settings=True):
459 """ 460 Returns a dictionary containing all section_settings for a given 461 section_name by first loading the settings in the config, loading 462 the defaults for all settings not specified, and then checking 463 that all required options have been specified 464 """ 465 store = AttributeDict() 466 self._load_settings(section_name, section_settings, store, 467 filter_settings) 468 self._load_defaults(section_settings, store) 469 self._check_required(section_name, section_settings, store) 470 return store
471
472 - def _get_section_name(self, section):
473 """ 474 Returns section name minus prefix 475 e.g. 476 $ print self._get_section('cluster smallcluster') 477 $ smallcluster 478 """ 479 return section.split()[1]
480
481 - def _get_sections(self, section_prefix):
482 """ 483 Returns all sections starting with section_prefix 484 e.g. 485 $ print self._get_sections('cluster') 486 $ ['cluster smallcluster', 'cluster mediumcluster', ..] 487 """ 488 return [s for s in self.config.sections() if 489 s.startswith(section_prefix)]
490
491 - def _load_sections(self, section_prefix, section_settings, 492 filter_settings=True):
493 """ 494 Loads all sections starting with section_prefix and returns a 495 dictionary containing the name and dictionary of settings for each 496 section. 497 keys --> section name (as returned by self._get_section_name) 498 values --> dictionary of settings for a given section 499 500 e.g. 501 $ print self._load_sections('volumes', self.plugin_settings) 502 503 {'myvol': {'__name__': 'volume myvol', 504 'device': None, 505 'mount_path': '/home', 506 'partition': 1, 507 'volume_id': 'vol-999999'}, 508 'myvol2': {'__name__': 'volume myvol2', 509 'device': None, 510 'mount_path': '/myvol2', 511 'partition': 1, 512 'volume_id': 'vol-999999'}, 513 """ 514 sections = self._get_sections(section_prefix) 515 sections_store = AttributeDict() 516 for sec in sections: 517 name = self._get_section_name(sec) 518 sections_store[name] = self._load_section(sec, section_settings, 519 filter_settings) 520 return sections_store
521
522 - def _load_cluster_sections(self, cluster_sections):
523 """ 524 Loads all cluster sections. Similar to _load_sections but also handles 525 populating specified keypair, volume, plugins, permissions, etc. 526 settings 527 """ 528 clusters = cluster_sections 529 cluster_store = AttributeDict() 530 for cl in clusters: 531 name = self._get_section_name(cl) 532 cluster_store[name] = AttributeDict() 533 self._load_settings(cl, self.cluster_settings, cluster_store[name]) 534 for cl in clusters: 535 name = self._get_section_name(cl) 536 self._load_extends_settings(name, cluster_store) 537 self._load_defaults(self.cluster_settings, cluster_store[name]) 538 self._load_keypairs(cluster_store[name]) 539 self._load_volumes(cluster_store[name]) 540 self._load_plugins(cluster_store[name]) 541 self._load_permissions(cluster_store[name]) 542 self._load_instance_types(cluster_store[name]) 543 self._check_required(cl, self.cluster_settings, 544 cluster_store[name]) 545 return cluster_store
546
547 - def load(self):
548 """ 549 Populate this config object from the StarCluster config 550 """ 551 log.debug('Loading config') 552 try: 553 self.globals = self._load_section('global', self.global_settings) 554 except exception.ConfigSectionMissing: 555 pass 556 try: 557 self.aws = self._load_section('aws info', self.aws_settings) 558 except exception.ConfigSectionMissing: 559 log.warn("no [aws info] section found in config") 560 log.warn("attempting to load credentials from environment...") 561 self.aws.update(self.get_aws_from_environ()) 562 self.keys = self._load_sections('key', self.key_settings) 563 self.vols = self._load_sections('volume', self.volume_settings) 564 self.plugins = self._load_sections('plugin', self.plugin_settings, 565 filter_settings=False) 566 self.permissions = self._load_sections('permission', 567 self.permission_settings) 568 sections = self._get_sections('cluster') 569 self.clusters = self._load_cluster_sections(sections) 570 return self
571
572 - def get_aws_from_environ(self):
573 """ 574 Returns AWS credentials defined in the user's shell 575 environment. 576 """ 577 awscreds = {} 578 for key in static.AWS_SETTINGS: 579 if key.upper() in os.environ: 580 awscreds[key] = os.environ.get(key.upper()) 581 elif key in os.environ: 582 awscreds[key] = os.environ.get(key) 583 return awscreds
584
585 - def get_aws_credentials(self):
586 """ 587 Returns AWS credentials defined in the configuration 588 file. Defining any of the AWS settings in the environment 589 overrides the configuration file. 590 """ 591 # first override with environment settings if they exist 592 self.aws.update(self.get_aws_from_environ()) 593 return self.aws
594
595 - def get_cluster_names(self):
596 return self.clusters
597
598 - def get_cluster_template(self, template_name, tag_name=None, 599 ec2_conn=None):
600 """ 601 Returns Cluster instance configured with the settings in the 602 config file. 603 604 template_name is the name of a cluster section defined in the config 605 606 tag_name, if specified, will be passed to Cluster instance 607 as cluster_tag 608 """ 609 try: 610 kwargs = {} 611 if tag_name: 612 kwargs.update(dict(cluster_tag=tag_name)) 613 kwargs.update(self.clusters[template_name]) 614 if not ec2_conn: 615 ec2_conn = self.get_easy_ec2() 616 clust = Cluster(ec2_conn, **kwargs) 617 return clust 618 except KeyError: 619 raise exception.ClusterTemplateDoesNotExist(template_name)
620
622 """ 623 Returns the default_template specified in the [global] section 624 of the config. Raises NoDefaultTemplateFound if no default cluster 625 template has been specified in the config. 626 """ 627 default = self.globals.get('default_template') 628 if not default: 629 raise exception.NoDefaultTemplateFound( 630 options=self.clusters.keys()) 631 if not default in self.clusters: 632 raise exception.ClusterTemplateDoesNotExist(default) 633 return default
634
635 - def get_clusters(self):
636 clusters = [] 637 for cl in self.clusters: 638 cl.append(self.get_cluster_template(cluster)) 639 return clusters
640
641 - def get_plugin(self, plugin):
642 try: 643 return self.plugins[plugin] 644 except KeyError: 645 raise exception.PluginNotFound(plugin)
646
647 - def get_key(self, keyname):
648 try: 649 return self.keys[keyname] 650 except KeyError: 651 raise exception.KeyNotFound(keyname)
652
653 - def get_easy_s3(self):
654 """ 655 Factory for EasyEC2 class that attempts to load AWS credentials from 656 the StarCluster config file. Returns an EasyS3 object if 657 successful. 658 """ 659 aws = self.get_aws_credentials() 660 try: 661 s3 = awsutils.EasyS3(**aws) 662 return s3 663 except TypeError: 664 raise exception.ConfigError("no aws credentials found")
665
666 - def get_easy_ec2(self):
667 """ 668 Factory for EasyEC2 class that attempts to load AWS credentials from 669 the StarCluster config file. Returns an EasyEC2 object if 670 successful. 671 """ 672 aws = self.get_aws_credentials() 673 try: 674 ec2 = awsutils.EasyEC2(**aws) 675 return ec2 676 except TypeError: 677 raise exception.ConfigError("no aws credentials found")
678
679 - def get_cluster_manager(self):
680 ec2 = self.get_easy_ec2() 681 return cluster.ClusterManager(self, ec2)
682 683 684 if __name__ == "__main__": 685 from pprint import pprint 686 cfg = StarClusterConfig().load() 687 pprint(cfg.aws) 688 pprint(cfg.clusters) 689 pprint(cfg.keys) 690 pprint(cfg.vols) 691