Package starcluster :: Module config
[hide private]
[frames] | no frames]

Source Code for Module starcluster.config

  1  #!/usr/bin/env python 
  2  import os 
  3  import urllib 
  4  import ConfigParser 
  5   
  6  from starcluster import utils 
  7  from starcluster import static 
  8  from starcluster import awsutils 
  9  from starcluster import exception 
 10  from starcluster.cluster import Cluster 
 11  from starcluster.utils import AttributeDict 
 12   
 13  from starcluster.logger import log 
 14   
 15  DEBUG_CONFIG = False 
16 17 18 -def get_easy_s3(config_file=None, cache=False):
19 """ 20 Factory for EasyS3 class that attempts to load AWS credentials from 21 the StarCluster config file. Returns an EasyS3 object if 22 successful. 23 """ 24 cfg = get_config(config_file, cache) 25 return cfg.get_easy_s3()
26
27 28 -def get_easy_ec2(config_file=None, cache=False):
29 """ 30 Factory for EasyEC2 class that attempts to load AWS credentials from 31 the StarCluster config file. Returns an EasyEC2 object if 32 successful. 33 """ 34 cfg = get_config(config_file, cache) 35 return cfg.get_easy_ec2()
36
37 38 -def get_config(config_file=None, cache=False):
39 """Factory for StarClusterConfig object""" 40 return StarClusterConfig(config_file, cache)
41
42 43 -class StarClusterConfig(object):
44 """ 45 Loads StarCluster configuration settings defined in config_file 46 which defaults to ~/.starclustercfg 47 48 Settings are available as follows: 49 50 cfg = StarClusterConfig() 51 or 52 cfg = StarClusterConfig('/path/to/my/config.cfg') 53 cfg.load() 54 aws_info = cfg.aws 55 cluster_cfg = cfg.clusters['mycluster'] 56 key_cfg = cfg.keys['gsg-keypair'] 57 print cluster_cfg 58 """ 59 60 global_settings = static.GLOBAL_SETTINGS 61 aws_settings = static.AWS_SETTINGS 62 key_settings = static.KEY_SETTINGS 63 volume_settings = static.EBS_VOLUME_SETTINGS 64 plugin_settings = static.PLUGIN_SETTINGS 65 cluster_settings = static.CLUSTER_SETTINGS 66 permission_settings = static.PERMISSION_SETTINGS 67 68 # until i can find a way to query AWS for instance types... 69 instance_types = static.INSTANCE_TYPES 70
71 - def __init__(self, config_file=None, cache=False):
72 self.cfg_file = config_file or static.STARCLUSTER_CFG_FILE 73 self.type_validators = { 74 int: self._get_int, 75 float: self._get_float, 76 str: self._get_string, 77 bool: self._get_bool, 78 list: self._get_list, 79 } 80 self._config = None 81 self.globals = AttributeDict() 82 self.aws = AttributeDict() 83 self.clusters = AttributeDict() 84 self.keys = AttributeDict() 85 self.vols = AttributeDict() 86 self.plugins = AttributeDict() 87 self.permissions = AttributeDict() 88 self.cache = cache
89
90 - def __repr__(self):
91 return "<StarClusterConfig: %s>" % self.cfg_file
92
93 - def _get_urlfp(self, url):
94 log.debug("Loading url: %s" % url) 95 try: 96 fp = urllib.urlopen(url) 97 if fp.getcode() == 404: 98 raise exception.ConfigError("url %s does not exist" % url) 99 fp.name = url 100 return fp 101 except IOError, e: 102 raise exception.ConfigError( 103 "error loading config from url %s\n%s" % (url, e))
104
105 - def _get_fp(self, cfg_file):
106 if not os.path.isdir(static.STARCLUSTER_CFG_DIR): 107 os.makedirs(static.STARCLUSTER_CFG_DIR) 108 log.debug("Loading file: %s" % cfg_file) 109 if os.path.exists(cfg_file): 110 if not os.path.isfile(cfg_file): 111 raise exception.ConfigError( 112 'config %s exists but is not a regular file' % cfg_file) 113 else: 114 raise exception.ConfigNotFound( 115 ("config file %s does not exist\n") % 116 cfg_file, cfg_file, 117 ) 118 return open(cfg_file)
119
120 - def _get_bool(self, config, section, option):
121 try: 122 opt = config.getboolean(section, option) 123 return opt 124 except ConfigParser.NoSectionError: 125 pass 126 except ConfigParser.NoOptionError: 127 pass 128 except ValueError: 129 raise exception.ConfigError( 130 "Expected True/False value for setting %s in section [%s]" % 131 (option, section))
132
133 - def _get_int(self, config, section, option):
134 try: 135 opt = config.getint(section, option) 136 return opt 137 except ConfigParser.NoSectionError: 138 pass 139 except ConfigParser.NoOptionError: 140 pass 141 except ValueError: 142 raise exception.ConfigError( 143 "Expected integer value for setting %s in section [%s]" % 144 (option, section))
145
146 - def _get_float(self, config, section, option):
147 try: 148 opt = config.getfloat(section, option) 149 return opt 150 except ConfigParser.NoSectionError: 151 pass 152 except ConfigParser.NoOptionError: 153 pass 154 except ValueError: 155 raise exception.ConfigError( 156 "Expected float value for setting %s in section [%s]" % 157 (option, section))
158
159 - def _get_string(self, config, section, option):
160 try: 161 opt = config.get(section, option) 162 return opt 163 except ConfigParser.NoSectionError: 164 pass 165 except ConfigParser.NoOptionError: 166 pass
167
168 - def _get_list(self, config, section, option):
169 val = self._get_string(config, section, option) 170 if val: 171 val = [v.strip() for v in val.split(',')] 172 return val
173
174 - def __load_config(self):
175 """ 176 Populates self._config with a new ConfigParser instance 177 """ 178 cfg = self.cfg_file 179 if utils.is_url(cfg): 180 cfg = self._get_urlfp(cfg) 181 else: 182 cfg = self._get_fp(cfg) 183 try: 184 cp = ConfigParser.ConfigParser() 185 cp.readfp(cfg) 186 return cp 187 except ConfigParser.MissingSectionHeaderError: 188 raise exception.ConfigHasNoSections(cfg.name) 189 except ConfigParser.ParsingError, e: 190 raise exception.ConfigError(e)
191
192 - def reload(self):
193 """ 194 Reloads the configuration file 195 """ 196 self._config = self.__load_config() 197 return self.load()
198 199 @property
200 - def config(self):
201 if self._config is None: 202 self._config = self.__load_config() 203 return self._config
204
205 - def _load_settings(self, section_name, settings, store, 206 filter_settings=True):
207 """ 208 Load section settings into a dictionary 209 """ 210 section = self.config._sections.get(section_name) 211 if not section: 212 raise exception.ConfigSectionMissing( 213 'Missing section %s in config' % section_name) 214 store.update(section) 215 section_conf = store 216 for setting in settings: 217 requirements = settings[setting] 218 func, required, default, options, callback = requirements 219 func = self.type_validators.get(func) 220 value = func(self.config, section_name, setting) 221 if value is not None: 222 if options and not value in options: 223 raise exception.ConfigError( 224 '"%s" setting in section "%s" must be one of: %s' % 225 (setting, section_name, 226 ', '.join([str(o) for o in options]))) 227 if callback: 228 value = callback(value) 229 section_conf[setting] = value 230 if filter_settings: 231 for key in store.keys(): 232 if key not in settings and key != '__name__': 233 store.pop(key)
234
235 - def _check_required(self, section_name, settings, store):
236 """ 237 Check that all required settings were specified in the config. 238 Raises ConfigError otherwise. 239 240 Note that if a setting specified has required=True and 241 default is not None then this method will not raise an error 242 because a default was given. In short, if a setting is required 243 you must provide None as the 'default' value. 244 """ 245 section_conf = store 246 for setting in settings: 247 requirements = settings[setting] 248 required = requirements[1] 249 value = section_conf.get(setting) 250 if value is None and required: 251 raise exception.ConfigError( 252 'missing required option %s in section "%s"' % 253 (setting, section_name))
254
255 - def _load_defaults(self, settings, store):
256 """ 257 Sets the default for each setting in settings regardless of whether 258 the setting was specified in the config or not. 259 """ 260 section_conf = store 261 for setting in settings: 262 default = settings[setting][2] 263 if section_conf.get(setting) is None: 264 if DEBUG_CONFIG: 265 log.debug('%s setting not specified. Defaulting to %s' % \ 266 (setting, default)) 267 section_conf[setting] = default
268
269 - def _load_extends_settings(self, section_name, store):
270 """ 271 Loads all settings from other template(s) specified by a section's 272 'extends' setting. 273 274 This method walks a dependency tree of sections from bottom up. Each 275 step is a group of settings for a section in the form of a dictionary. 276 A 'master' dictionary is updated with the settings at each step. This 277 causes the next group of settings to override the previous, and so on. 278 The 'section_name' settings are at the top of the dep tree. 279 """ 280 section = store[section_name] 281 extends = section.get('extends') 282 if extends is None: 283 return 284 if DEBUG_CONFIG: 285 log.debug('%s extends %s' % (section_name, extends)) 286 extensions = [section] 287 while True: 288 extends = section.get('extends', None) 289 if not extends: 290 break 291 try: 292 section = store[extends] 293 if section in extensions: 294 exts = ', '.join([self._get_section_name(x['__name__']) 295 for x in extensions]) 296 raise exception.ConfigError( 297 ("Cyclical dependency between sections %s. " % exts) \ 298 + "Check your extends settings.") 299 extensions.insert(0, section) 300 except KeyError: 301 raise exception.ConfigError( 302 "%s can't extend non-existent section %s" % \ 303 (section_name, extends)) 304 transform = AttributeDict() 305 for extension in extensions: 306 transform.update(extension) 307 store[section_name] = transform
308
309 - def _load_keypairs(self, store):
310 cluster_section = store 311 keyname = cluster_section.get('keyname') 312 if not keyname: 313 return 314 keypair = self.keys.get(keyname) 315 if keypair is None: 316 raise exception.ConfigError( 317 "keypair '%s' not defined in config" % keyname) 318 cluster_section['keyname'] = keyname 319 cluster_section['key_location'] = keypair.get('key_location')
320
321 - def _load_volumes(self, store):
322 cluster_section = store 323 volumes = cluster_section.get('volumes') 324 if not volumes or isinstance(volumes, AttributeDict): 325 return 326 vols = AttributeDict() 327 cluster_section['volumes'] = vols 328 for volume in volumes: 329 if not volume in self.vols: 330 raise exception.ConfigError( 331 "volume '%s' not defined in config" % volume) 332 vol = self.vols.get(volume) 333 vols[volume] = vol
334
335 - def _load_plugins(self, store):
336 cluster_section = store 337 plugins = cluster_section.get('plugins') 338 if not plugins or isinstance(plugins[0], AttributeDict): 339 return 340 plugs = [] 341 cluster_section['plugins'] = plugs 342 for plugin in plugins: 343 if plugin in self.plugins: 344 p = self.plugins.get(plugin) 345 p['__name__'] = p['__name__'].split()[-1] 346 plugs.append(p) 347 else: 348 raise exception.ConfigError( 349 "plugin '%s' not defined in config" % plugin)
350
351 - def _load_permissions(self, store):
352 cluster_section = store 353 permissions = cluster_section.get('permissions') 354 if not permissions or isinstance(permissions, AttributeDict): 355 return 356 perms = AttributeDict() 357 cluster_section['permissions'] = perms 358 for perm in permissions: 359 if perm in self.permissions: 360 p = self.permissions.get(perm) 361 p['__name__'] = p['__name__'].split()[-1] 362 perms[perm] = p 363 else: 364 raise exception.ConfigError( 365 "permission '%s' not defined in config" % perm)
366
367 - def _load_instance_types(self, store):
368 cluster_section = store 369 instance_types = cluster_section.get('node_instance_type') 370 if isinstance(instance_types, basestring): 371 return 372 itypes = [] 373 cluster_section['node_instance_types'] = itypes 374 total_num_nodes = 0 375 choices_string = ', '.join(static.INSTANCE_TYPES.keys()) 376 try: 377 default_instance_type = instance_types[-1] 378 if not default_instance_type in static.INSTANCE_TYPES: 379 raise exception.ConfigError( 380 ("invalid node_instance_type specified: '%s'\n" + 381 "must be one of: %s") % 382 (default_instance_type, choices_string)) 383 except IndexError: 384 default_instance_type = None 385 cluster_section['node_instance_type'] = default_instance_type 386 for type_spec in instance_types[:-1]: 387 type_spec = type_spec.split(':') 388 if len(type_spec) > 3: 389 raise exception.ConfigError( 390 "invalid node_instance_type item specified: %s" % \ 391 type_spec) 392 itype = type_spec[0] 393 itype_image = None 394 itype_num = 1 395 if not itype in static.INSTANCE_TYPES: 396 raise exception.ConfigError( 397 ("invalid type specified (%s) in node_instance_type " + \ 398 "item: '%s'\nmust be one of: %s") % 399 (itype, type_spec, choices_string)) 400 if len(type_spec) == 2: 401 itype, next_var = type_spec 402 try: 403 itype_num = int(next_var) 404 except (TypeError, ValueError): 405 itype_image = next_var 406 elif len(type_spec) == 3: 407 itype, itype_image, itype_num = type_spec 408 try: 409 itype_num = int(itype_num) 410 if itype_num < 1: 411 raise TypeError 412 total_num_nodes += itype_num 413 except (ValueError, TypeError): 414 raise exception.ConfigError( 415 ("number of instances (%s) of type '%s' must " + \ 416 "be an integer > 1") % (itype_num, itype)) 417 itype_dic = AttributeDict(size=itype_num, image=itype_image, 418 type=itype) 419 itypes.append(itype_dic)
420
421 - def _load_section(self, section_name, section_settings, 422 filter_settings=True):
423 """ 424 Returns a dictionary containing all section_settings for a given 425 section_name by first loading the settings in the config, loading 426 the defaults for all settings not specified, and then checking 427 that all required options have been specified 428 """ 429 store = AttributeDict() 430 self._load_settings(section_name, section_settings, store, 431 filter_settings) 432 self._load_defaults(section_settings, store) 433 self._check_required(section_name, section_settings, store) 434 return store
435
436 - def _get_section_name(self, section):
437 """ 438 Returns section name minus prefix 439 e.g. 440 $ print self._get_section('cluster smallcluster') 441 $ smallcluster 442 """ 443 return section.split()[1]
444
445 - def _get_sections(self, section_prefix):
446 """ 447 Returns all sections starting with section_prefix 448 e.g. 449 $ print self._get_sections('cluster') 450 $ ['cluster smallcluster', 'cluster mediumcluster', ..] 451 """ 452 return [s for s in self.config.sections() if 453 s.startswith(section_prefix)]
454
455 - def _load_sections(self, section_prefix, section_settings, 456 filter_settings=True):
457 """ 458 Loads all sections starting with section_prefix and returns a 459 dictionary containing the name and dictionary of settings for each 460 section. 461 keys --> section name (as returned by self._get_section_name) 462 values --> dictionary of settings for a given section 463 464 e.g. 465 $ print self._load_sections('volumes', self.plugin_settings) 466 467 {'myvol': {'__name__': 'volume myvol', 468 'device': None, 469 'mount_path': '/home', 470 'partition': 1, 471 'volume_id': 'vol-999999'}, 472 'myvol2': {'__name__': 'volume myvol2', 473 'device': None, 474 'mount_path': '/myvol2', 475 'partition': 1, 476 'volume_id': 'vol-999999'}, 477 """ 478 sections = self._get_sections(section_prefix) 479 sections_store = AttributeDict() 480 for sec in sections: 481 name = self._get_section_name(sec) 482 sections_store[name] = self._load_section(sec, section_settings, 483 filter_settings) 484 return sections_store
485
486 - def _load_cluster_sections(self, cluster_sections):
487 """ 488 Loads all cluster sections. Similar to _load_sections but also handles 489 populating specified keypair,volume,plugins,permissions,etc settings 490 """ 491 clusters = cluster_sections 492 cluster_store = AttributeDict() 493 for cluster in clusters: 494 name = self._get_section_name(cluster) 495 cluster_store[name] = AttributeDict() 496 self._load_settings(cluster, self.cluster_settings, 497 cluster_store[name]) 498 for cluster in clusters: 499 name = self._get_section_name(cluster) 500 self._load_extends_settings(name, cluster_store) 501 self._load_defaults(self.cluster_settings, cluster_store[name]) 502 self._load_keypairs(cluster_store[name]) 503 self._load_volumes(cluster_store[name]) 504 self._load_plugins(cluster_store[name]) 505 self._load_permissions(cluster_store[name]) 506 self._load_instance_types(cluster_store[name]) 507 self._check_required(cluster, self.cluster_settings, 508 cluster_store[name]) 509 return cluster_store
510
511 - def load(self):
512 """ 513 Populate this config object from the StarCluster config 514 """ 515 log.debug('Loading config') 516 try: 517 self.globals = self._load_section('global', self.global_settings) 518 except exception.ConfigSectionMissing: 519 pass 520 try: 521 self.aws = self._load_section('aws info', self.aws_settings) 522 except exception.ConfigSectionMissing: 523 log.warn("no [aws info] section found in config") 524 log.warn("attempting to load credentials from environment...") 525 self.aws.update(self.get_aws_from_environ()) 526 self.keys = self._load_sections('key', self.key_settings) 527 self.vols = self._load_sections('volume', self.volume_settings) 528 self.plugins = self._load_sections('plugin', self.plugin_settings, 529 filter_settings=False) 530 self.permissions = self._load_sections('permission', 531 self.permission_settings) 532 sections = self._get_sections('cluster') 533 self.clusters = self._load_cluster_sections(sections) 534 return self
535
536 - def get_aws_from_environ(self):
537 """ 538 Returns AWS credentials defined in the user's shell 539 environment. 540 """ 541 awscreds = {} 542 for key in static.AWS_SETTINGS: 543 if key.upper() in os.environ: 544 awscreds[key] = os.environ.get(key.upper()) 545 elif key in os.environ: 546 awscreds[key] = os.environ.get(key) 547 return awscreds
548
549 - def get_aws_credentials(self):
550 """ 551 Returns AWS credentials defined in the configuration 552 file. Defining any of the AWS settings in the environment 553 overrides the configuration file. 554 """ 555 # first override with environment settings if they exist 556 self.aws.update(self.get_aws_from_environ()) 557 return self.aws
558
559 - def get_cluster_names(self):
560 return self.clusters
561
562 - def get_cluster_template(self, template_name, tag_name=None, 563 ec2_conn=None):
564 """ 565 Returns Cluster instance configured with the settings in the 566 config file. 567 568 template_name is the name of a cluster section defined in the config 569 570 tag_name, if specified, will be passed to Cluster instance 571 as cluster_tag 572 """ 573 try: 574 kwargs = {} 575 if tag_name: 576 kwargs.update(dict(cluster_tag=tag_name)) 577 kwargs.update(self.clusters[template_name]) 578 if not ec2_conn: 579 ec2_conn = self.get_easy_ec2() 580 clust = Cluster(ec2_conn, **kwargs) 581 return clust 582 except KeyError: 583 raise exception.ClusterTemplateDoesNotExist(template_name)
584
586 """ 587 Returns the default_template specified in the [global] section 588 of the config. Raises NoDefaultTemplateFound if no default cluster 589 template has been specified in the config. 590 """ 591 default = self.globals.get('default_template') 592 if not default: 593 raise exception.NoDefaultTemplateFound( 594 options=self.clusters.keys()) 595 if not default in self.clusters: 596 raise exception.ClusterTemplateDoesNotExist(default) 597 return default
598
599 - def get_clusters(self):
600 clusters = [] 601 for cluster in self.clusters: 602 clusters.append(self.get_cluster_template(cluster)) 603 return clusters
604
605 - def get_plugin(self, plugin):
606 try: 607 return self.plugins[plugin] 608 except KeyError: 609 raise exception.PluginNotFound(plugin)
610
611 - def get_key(self, keyname):
612 try: 613 return self.keys[keyname] 614 except KeyError: 615 raise exception.KeyNotFound(keyname)
616
617 - def get_easy_s3(self):
618 """ 619 Factory for EasyEC2 class that attempts to load AWS credentials from 620 the StarCluster config file. Returns an EasyS3 object if 621 successful. 622 """ 623 aws = self.get_aws_credentials() 624 try: 625 s3 = awsutils.EasyS3(**aws) 626 return s3 627 except TypeError: 628 raise exception.ConfigError("no aws credentials found")
629
630 - def get_easy_ec2(self):
631 """ 632 Factory for EasyEC2 class that attempts to load AWS credentials from 633 the StarCluster config file. Returns an EasyEC2 object if 634 successful. 635 """ 636 aws = self.get_aws_credentials() 637 try: 638 ec2 = awsutils.EasyEC2(**aws) 639 return ec2 640 except TypeError: 641 raise exception.ConfigError("no aws credentials found")
642 643 if __name__ == "__main__": 644 from pprint import pprint 645 cfg = StarClusterConfig().load() 646 pprint(cfg.aws) 647 pprint(cfg.clusters) 648 pprint(cfg.keys) 649 pprint(cfg.vols) 650