Package starcluster :: Module config
[hide private]
[frames] | no frames]

Source Code for Module starcluster.config

  1  #!/usr/bin/env python 
  2  import os 
  3  import urllib 
  4  import ConfigParser 
  5   
  6  from starcluster import utils 
  7  from starcluster import static 
  8  from starcluster import cluster 
  9  from starcluster import awsutils 
 10  from starcluster import exception 
 11  from starcluster.cluster import Cluster 
 12  from starcluster.utils import AttributeDict 
 13   
 14  from starcluster.logger import log 
 15   
 16  DEBUG_CONFIG = False 
17 18 19 -def get_easy_s3(config_file=None, cache=False):
20 """ 21 Factory for EasyS3 class that attempts to load AWS credentials from 22 the StarCluster config file. Returns an EasyS3 object if 23 successful. 24 """ 25 cfg = get_config(config_file, cache) 26 return cfg.get_easy_s3()
27
28 29 -def get_easy_ec2(config_file=None, cache=False):
30 """ 31 Factory for EasyEC2 class that attempts to load AWS credentials from 32 the StarCluster config file. Returns an EasyEC2 object if 33 successful. 34 """ 35 cfg = get_config(config_file, cache) 36 return cfg.get_easy_ec2()
37
38 39 -def get_cluster_manager(config_file=None, cache=False):
40 """ 41 Factory for ClusterManager class that attempts to load AWS credentials from 42 the StarCluster config file. Returns a ClusterManager object if successful 43 """ 44 cfg = get_config(config_file, cache) 45 return cfg.get_cluster_manager()
46
47 48 -def get_config(config_file=None, cache=False):
49 """Factory for StarClusterConfig object""" 50 return StarClusterConfig(config_file, cache)
51
52 53 -class StarClusterConfig(object):
54 """ 55 Loads StarCluster configuration settings defined in config_file 56 which defaults to ~/.starclustercfg 57 58 Settings are available as follows: 59 60 cfg = StarClusterConfig() 61 or 62 cfg = StarClusterConfig('/path/to/my/config.cfg') 63 cfg.load() 64 aws_info = cfg.aws 65 cluster_cfg = cfg.clusters['mycluster'] 66 key_cfg = cfg.keys['gsg-keypair'] 67 print cluster_cfg 68 """ 69 70 global_settings = static.GLOBAL_SETTINGS 71 aws_settings = static.AWS_SETTINGS 72 key_settings = static.KEY_SETTINGS 73 volume_settings = static.EBS_VOLUME_SETTINGS 74 plugin_settings = static.PLUGIN_SETTINGS 75 cluster_settings = static.CLUSTER_SETTINGS 76 permission_settings = static.PERMISSION_SETTINGS 77 78 # until i can find a way to query AWS for instance types... 79 instance_types = static.INSTANCE_TYPES 80
81 - def __init__(self, config_file=None, cache=False):
82 self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE 83 self.type_validators = { 84 int: self._get_int, 85 float: self._get_float, 86 str: self._get_string, 87 bool: self._get_bool, 88 list: self._get_list, 89 } 90 self._config = None 91 self.globals = AttributeDict() 92 self.aws = AttributeDict() 93 self.clusters = AttributeDict() 94 self.keys = AttributeDict() 95 self.vols = AttributeDict() 96 self.plugins = AttributeDict() 97 self.permissions = AttributeDict() 98 self.cache = cache
99
100 - def __repr__(self):
101 return "<StarClusterConfig: %s>" % self.cfg_file
102
103 - def _get_urlfp(self, url):
104 log.debug("Loading url: %s" % url) 105 try: 106 fp = urllib.urlopen(url) 107 if fp.getcode() == 404: 108 raise exception.ConfigError("url %s does not exist" % url) 109 fp.name = url 110 return fp 111 except IOError, e: 112 raise exception.ConfigError( 113 "error loading config from url %s\n%s" % (url, e))
114
115 - def _get_fp(self, cfg_file):
116 if not os.path.isdir(static.STARCLUSTER_CFG_DIR): 117 os.makedirs(static.STARCLUSTER_CFG_DIR) 118 log.debug("Loading file: %s" % cfg_file) 119 if os.path.exists(cfg_file): 120 if not os.path.isfile(cfg_file): 121 raise exception.ConfigError( 122 'config %s exists but is not a regular file' % cfg_file) 123 else: 124 raise exception.ConfigNotFound( 125 ("config file %s does not exist\n") % 126 cfg_file, cfg_file, 127 ) 128 return open(cfg_file)
129
130 - def _get_bool(self, config, section, option):
131 try: 132 opt = config.getboolean(section, option) 133 return opt 134 except ConfigParser.NoSectionError: 135 pass 136 except ConfigParser.NoOptionError: 137 pass 138 except ValueError: 139 raise exception.ConfigError( 140 "Expected True/False value for setting %s in section [%s]" % 141 (option, section))
142
143 - def _get_int(self, config, section, option):
144 try: 145 opt = config.getint(section, option) 146 return opt 147 except ConfigParser.NoSectionError: 148 pass 149 except ConfigParser.NoOptionError: 150 pass 151 except ValueError: 152 raise exception.ConfigError( 153 "Expected integer value for setting %s in section [%s]" % 154 (option, section))
155
156 - def _get_float(self, config, section, option):
157 try: 158 opt = config.getfloat(section, option) 159 return opt 160 except ConfigParser.NoSectionError: 161 pass 162 except ConfigParser.NoOptionError: 163 pass 164 except ValueError: 165 raise exception.ConfigError( 166 "Expected float value for setting %s in section [%s]" % 167 (option, section))
168
169 - def _get_string(self, config, section, option):
170 try: 171 opt = config.get(section, option) 172 return opt 173 except ConfigParser.NoSectionError: 174 pass 175 except ConfigParser.NoOptionError: 176 pass
177
178 - def _get_list(self, config, section, option):
179 val = self._get_string(config, section, option) 180 if val: 181 val = [v.strip() for v in val.split(',')] 182 return val
183
184 - def __load_config(self):
185 """ 186 Populates self._config with a new ConfigParser instance 187 """ 188 cfg = self.cfg_file 189 if utils.is_url(cfg): 190 cfg = self._get_urlfp(cfg) 191 else: 192 cfg = self._get_fp(cfg) 193 try: 194 cp = ConfigParser.ConfigParser() 195 cp.readfp(cfg) 196 return cp 197 except ConfigParser.MissingSectionHeaderError: 198 raise exception.ConfigHasNoSections(cfg.name) 199 except ConfigParser.ParsingError, e: 200 raise exception.ConfigError(e)
201
202 - def reload(self):
203 """ 204 Reloads the configuration file 205 """ 206 self._config = self.__load_config() 207 return self.load()
208 209 @property
210 - def config(self):
211 if self._config is None: 212 self._config = self.__load_config() 213 return self._config
214
215 - def _load_settings(self, section_name, settings, store, 216 filter_settings=True):
217 """ 218 Load section settings into a dictionary 219 """ 220 section = self.config._sections.get(section_name) 221 if not section: 222 raise exception.ConfigSectionMissing( 223 'Missing section %s in config' % section_name) 224 store.update(section) 225 section_conf = store 226 for setting in settings: 227 requirements = settings[setting] 228 func, required, default, options, callback = requirements 229 func = self.type_validators.get(func) 230 value = func(self.config, section_name, setting) 231 if value is not None: 232 if options and not value in options: 233 raise exception.ConfigError( 234 '"%s" setting in section "%s" must be one of: %s' % 235 (setting, section_name, 236 ', '.join([str(o) for o in options]))) 237 if callback: 238 value = callback(value) 239 section_conf[setting] = value 240 if filter_settings: 241 for key in store.keys(): 242 if key not in settings and key != '__name__': 243 store.pop(key)
244
245 - def _check_required(self, section_name, settings, store):
246 """ 247 Check that all required settings were specified in the config. 248 Raises ConfigError otherwise. 249 250 Note that if a setting specified has required=True and 251 default is not None then this method will not raise an error 252 because a default was given. In short, if a setting is required 253 you must provide None as the 'default' value. 254 """ 255 section_conf = store 256 for setting in settings: 257 requirements = settings[setting] 258 required = requirements[1] 259 value = section_conf.get(setting) 260 if value is None and required: 261 raise exception.ConfigError( 262 'missing required option %s in section "%s"' % 263 (setting, section_name))
264
265 - def _load_defaults(self, settings, store):
266 """ 267 Sets the default for each setting in settings regardless of whether 268 the setting was specified in the config or not. 269 """ 270 section_conf = store 271 for setting in settings: 272 default = settings[setting][2] 273 if section_conf.get(setting) is None: 274 if DEBUG_CONFIG: 275 log.debug('%s setting not specified. Defaulting to %s' % \ 276 (setting, default)) 277 section_conf[setting] = default
278
279 - def _load_extends_settings(self, section_name, store):
280 """ 281 Loads all settings from other template(s) specified by a section's 282 'extends' setting. 283 284 This method walks a dependency tree of sections from bottom up. Each 285 step is a group of settings for a section in the form of a dictionary. 286 A 'master' dictionary is updated with the settings at each step. This 287 causes the next group of settings to override the previous, and so on. 288 The 'section_name' settings are at the top of the dep tree. 289 """ 290 section = store[section_name] 291 extends = section.get('extends') 292 if extends is None: 293 return 294 if DEBUG_CONFIG: 295 log.debug('%s extends %s' % (section_name, extends)) 296 extensions = [section] 297 while True: 298 extends = section.get('extends', None) 299 if not extends: 300 break 301 try: 302 section = store[extends] 303 if section in extensions: 304 exts = ', '.join([self._get_section_name(x['__name__']) 305 for x in extensions]) 306 raise exception.ConfigError( 307 ("Cyclical dependency between sections %s. " % exts) \ 308 + "Check your extends settings.") 309 extensions.insert(0, section) 310 except KeyError: 311 raise exception.ConfigError( 312 "%s can't extend non-existent section %s" % \ 313 (section_name, extends)) 314 transform = AttributeDict() 315 for extension in extensions: 316 transform.update(extension) 317 store[section_name] = transform
318
319 - def _load_keypairs(self, store):
320 cluster_section = store 321 keyname = cluster_section.get('keyname') 322 if not keyname: 323 return 324 keypair = self.keys.get(keyname) 325 if keypair is None: 326 raise exception.ConfigError( 327 "keypair '%s' not defined in config" % keyname) 328 cluster_section['keyname'] = keyname 329 cluster_section['key_location'] = keypair.get('key_location')
330
331 - def _load_volumes(self, store):
332 cluster_section = store 333 volumes = cluster_section.get('volumes') 334 if not volumes or isinstance(volumes, AttributeDict): 335 return 336 vols = AttributeDict() 337 cluster_section['volumes'] = vols 338 for volume in volumes: 339 if not volume in self.vols: 340 raise exception.ConfigError( 341 "volume '%s' not defined in config" % volume) 342 vol = self.vols.get(volume) 343 vols[volume] = vol
344
345 - def _load_plugins(self, store):
346 cluster_section = store 347 plugins = cluster_section.get('plugins') 348 if not plugins or isinstance(plugins[0], AttributeDict): 349 return 350 plugs = [] 351 cluster_section['plugins'] = plugs 352 for plugin in plugins: 353 if plugin in self.plugins: 354 p = self.plugins.get(plugin) 355 p['__name__'] = p['__name__'].split()[-1] 356 plugs.append(p) 357 else: 358 raise exception.ConfigError( 359 "plugin '%s' not defined in config" % plugin)
360
361 - def _load_permissions(self, store):
362 cluster_section = store 363 permissions = cluster_section.get('permissions') 364 if not permissions or isinstance(permissions, AttributeDict): 365 return 366 perms = AttributeDict() 367 cluster_section['permissions'] = perms 368 for perm in permissions: 369 if perm in self.permissions: 370 p = self.permissions.get(perm) 371 p['__name__'] = p['__name__'].split()[-1] 372 perms[perm] = p 373 else: 374 raise exception.ConfigError( 375 "permission '%s' not defined in config" % perm)
376
377 - def _load_instance_types(self, store):
378 cluster_section = store 379 instance_types = cluster_section.get('node_instance_type') 380 if isinstance(instance_types, basestring): 381 return 382 itypes = [] 383 cluster_section['node_instance_types'] = itypes 384 total_num_nodes = 0 385 choices_string = ', '.join(static.INSTANCE_TYPES.keys()) 386 try: 387 default_instance_type = instance_types[-1] 388 if not default_instance_type in static.INSTANCE_TYPES: 389 raise exception.ConfigError( 390 ("invalid node_instance_type specified: '%s'\n" + 391 "must be one of: %s") % 392 (default_instance_type, choices_string)) 393 except IndexError: 394 default_instance_type = None 395 cluster_section['node_instance_type'] = default_instance_type 396 for type_spec in instance_types[:-1]: 397 type_spec = type_spec.split(':') 398 if len(type_spec) > 3: 399 raise exception.ConfigError( 400 "invalid node_instance_type item specified: %s" % \ 401 type_spec) 402 itype = type_spec[0] 403 itype_image = None 404 itype_num = 1 405 if not itype in static.INSTANCE_TYPES: 406 raise exception.ConfigError( 407 ("invalid type specified (%s) in node_instance_type " + \ 408 "item: '%s'\nmust be one of: %s") % 409 (itype, type_spec, choices_string)) 410 if len(type_spec) == 2: 411 itype, next_var = type_spec 412 try: 413 itype_num = int(next_var) 414 except (TypeError, ValueError): 415 itype_image = next_var 416 elif len(type_spec) == 3: 417 itype, itype_image, itype_num = type_spec 418 try: 419 itype_num = int(itype_num) 420 if itype_num < 1: 421 raise TypeError 422 total_num_nodes += itype_num 423 except (ValueError, TypeError): 424 raise exception.ConfigError( 425 ("number of instances (%s) of type '%s' must " + \ 426 "be an integer > 1") % (itype_num, itype)) 427 itype_dic = AttributeDict(size=itype_num, image=itype_image, 428 type=itype) 429 itypes.append(itype_dic)
430
431 - def _load_section(self, section_name, section_settings, 432 filter_settings=True):
433 """ 434 Returns a dictionary containing all section_settings for a given 435 section_name by first loading the settings in the config, loading 436 the defaults for all settings not specified, and then checking 437 that all required options have been specified 438 """ 439 store = AttributeDict() 440 self._load_settings(section_name, section_settings, store, 441 filter_settings) 442 self._load_defaults(section_settings, store) 443 self._check_required(section_name, section_settings, store) 444 return store
445
446 - def _get_section_name(self, section):
447 """ 448 Returns section name minus prefix 449 e.g. 450 $ print self._get_section('cluster smallcluster') 451 $ smallcluster 452 """ 453 return section.split()[1]
454
455 - def _get_sections(self, section_prefix):
456 """ 457 Returns all sections starting with section_prefix 458 e.g. 459 $ print self._get_sections('cluster') 460 $ ['cluster smallcluster', 'cluster mediumcluster', ..] 461 """ 462 return [s for s in self.config.sections() if 463 s.startswith(section_prefix)]
464
465 - def _load_sections(self, section_prefix, section_settings, 466 filter_settings=True):
467 """ 468 Loads all sections starting with section_prefix and returns a 469 dictionary containing the name and dictionary of settings for each 470 section. 471 keys --> section name (as returned by self._get_section_name) 472 values --> dictionary of settings for a given section 473 474 e.g. 475 $ print self._load_sections('volumes', self.plugin_settings) 476 477 {'myvol': {'__name__': 'volume myvol', 478 'device': None, 479 'mount_path': '/home', 480 'partition': 1, 481 'volume_id': 'vol-999999'}, 482 'myvol2': {'__name__': 'volume myvol2', 483 'device': None, 484 'mount_path': '/myvol2', 485 'partition': 1, 486 'volume_id': 'vol-999999'}, 487 """ 488 sections = self._get_sections(section_prefix) 489 sections_store = AttributeDict() 490 for sec in sections: 491 name = self._get_section_name(sec) 492 sections_store[name] = self._load_section(sec, section_settings, 493 filter_settings) 494 return sections_store
495
496 - def _load_cluster_sections(self, cluster_sections):
497 """ 498 Loads all cluster sections. Similar to _load_sections but also handles 499 populating specified keypair,volume,plugins,permissions,etc settings 500 """ 501 clusters = cluster_sections 502 cluster_store = AttributeDict() 503 for cl in clusters: 504 name = self._get_section_name(cl) 505 cluster_store[name] = AttributeDict() 506 self._load_settings(cl, self.cluster_settings, cluster_store[name]) 507 for cl in clusters: 508 name = self._get_section_name(cl) 509 self._load_extends_settings(name, cluster_store) 510 self._load_defaults(self.cluster_settings, cluster_store[name]) 511 self._load_keypairs(cluster_store[name]) 512 self._load_volumes(cluster_store[name]) 513 self._load_plugins(cluster_store[name]) 514 self._load_permissions(cluster_store[name]) 515 self._load_instance_types(cluster_store[name]) 516 self._check_required(cl, self.cluster_settings, 517 cluster_store[name]) 518 return cluster_store
519
520 - def load(self):
521 """ 522 Populate this config object from the StarCluster config 523 """ 524 log.debug('Loading config') 525 try: 526 self.globals = self._load_section('global', self.global_settings) 527 except exception.ConfigSectionMissing: 528 pass 529 try: 530 self.aws = self._load_section('aws info', self.aws_settings) 531 except exception.ConfigSectionMissing: 532 log.warn("no [aws info] section found in config") 533 log.warn("attempting to load credentials from environment...") 534 self.aws.update(self.get_aws_from_environ()) 535 self.keys = self._load_sections('key', self.key_settings) 536 self.vols = self._load_sections('volume', self.volume_settings) 537 self.plugins = self._load_sections('plugin', self.plugin_settings, 538 filter_settings=False) 539 self.permissions = self._load_sections('permission', 540 self.permission_settings) 541 sections = self._get_sections('cluster') 542 self.clusters = self._load_cluster_sections(sections) 543 return self
544
545 - def get_aws_from_environ(self):
546 """ 547 Returns AWS credentials defined in the user's shell 548 environment. 549 """ 550 awscreds = {} 551 for key in static.AWS_SETTINGS: 552 if key.upper() in os.environ: 553 awscreds[key] = os.environ.get(key.upper()) 554 elif key in os.environ: 555 awscreds[key] = os.environ.get(key) 556 return awscreds
557
558 - def get_aws_credentials(self):
559 """ 560 Returns AWS credentials defined in the configuration 561 file. Defining any of the AWS settings in the environment 562 overrides the configuration file. 563 """ 564 # first override with environment settings if they exist 565 self.aws.update(self.get_aws_from_environ()) 566 return self.aws
567
568 - def get_cluster_names(self):
569 return self.clusters
570
571 - def get_cluster_template(self, template_name, tag_name=None, 572 ec2_conn=None):
573 """ 574 Returns Cluster instance configured with the settings in the 575 config file. 576 577 template_name is the name of a cluster section defined in the config 578 579 tag_name, if specified, will be passed to Cluster instance 580 as cluster_tag 581 """ 582 try: 583 kwargs = {} 584 if tag_name: 585 kwargs.update(dict(cluster_tag=tag_name)) 586 kwargs.update(self.clusters[template_name]) 587 if not ec2_conn: 588 ec2_conn = self.get_easy_ec2() 589 clust = Cluster(ec2_conn, **kwargs) 590 return clust 591 except KeyError: 592 raise exception.ClusterTemplateDoesNotExist(template_name)
593
595 """ 596 Returns the default_template specified in the [global] section 597 of the config. Raises NoDefaultTemplateFound if no default cluster 598 template has been specified in the config. 599 """ 600 default = self.globals.get('default_template') 601 if not default: 602 raise exception.NoDefaultTemplateFound( 603 options=self.clusters.keys()) 604 if not default in self.clusters: 605 raise exception.ClusterTemplateDoesNotExist(default) 606 return default
607
608 - def get_clusters(self):
609 clusters = [] 610 for cl in self.clusters: 611 cl.append(self.get_cluster_template(cluster)) 612 return clusters
613
614 - def get_plugin(self, plugin):
615 try: 616 return self.plugins[plugin] 617 except KeyError: 618 raise exception.PluginNotFound(plugin)
619
620 - def get_key(self, keyname):
621 try: 622 return self.keys[keyname] 623 except KeyError: 624 raise exception.KeyNotFound(keyname)
625
626 - def get_easy_s3(self):
627 """ 628 Factory for EasyEC2 class that attempts to load AWS credentials from 629 the StarCluster config file. Returns an EasyS3 object if 630 successful. 631 """ 632 aws = self.get_aws_credentials() 633 try: 634 s3 = awsutils.EasyS3(**aws) 635 return s3 636 except TypeError: 637 raise exception.ConfigError("no aws credentials found")
638
639 - def get_easy_ec2(self):
640 """ 641 Factory for EasyEC2 class that attempts to load AWS credentials from 642 the StarCluster config file. Returns an EasyEC2 object if 643 successful. 644 """ 645 aws = self.get_aws_credentials() 646 try: 647 ec2 = awsutils.EasyEC2(**aws) 648 return ec2 649 except TypeError: 650 raise exception.ConfigError("no aws credentials found")
651
652 - def get_cluster_manager(self):
653 ec2 = self.get_easy_ec2() 654 return cluster.ClusterManager(self, ec2)
655 656 657 if __name__ == "__main__": 658 from pprint import pprint 659 cfg = StarClusterConfig().load() 660 pprint(cfg.aws) 661 pprint(cfg.clusters) 662 pprint(cfg.keys) 663 pprint(cfg.vols) 664