Package starcluster :: Module awsutils
[hide private]
[frames] | no frames]

Source Code for Module starcluster.awsutils

   1  #!/usr/bin/env python 
   2  """ 
   3  EC2/S3 Utility Classes 
   4  """ 
   5   
   6  import os 
   7  import re 
   8  import time 
   9  import base64 
  10  import string 
  11  import tempfile 
  12   
  13  import boto 
  14  import boto.ec2 
  15  import boto.s3.connection 
  16   
  17  from starcluster import image 
  18  from starcluster import utils 
  19  from starcluster import static 
  20  from starcluster import webtools 
  21  from starcluster import exception 
  22  from starcluster import progressbar 
  23  from starcluster.utils import print_timing 
  24  from starcluster.logger import log 
25 26 27 -class EasyAWS(object):
28 - def __init__(self, aws_access_key_id, aws_secret_access_key, 29 connection_authenticator, **kwargs):
30 """ 31 Create an EasyAWS object. 32 33 Requires aws_access_key_id/aws_secret_access_key from an Amazon Web 34 Services (AWS) account and a connection_authenticator function that 35 returns an authenticated AWS connection object 36 37 Providing only the keys will default to using Amazon EC2 38 39 kwargs are passed to the connection_authenticator's constructor 40 """ 41 self.aws_access_key_id = aws_access_key_id 42 self.aws_secret_access_key = aws_secret_access_key 43 self.connection_authenticator = connection_authenticator 44 self._conn = None 45 self._kwargs = kwargs
46
47 - def reload(self):
48 self._conn = None 49 return self.conn
50 51 @property
52 - def conn(self):
53 if self._conn is None: 54 log.debug('creating self._conn w/ connection_authenticator ' + 55 'kwargs = %s' % self._kwargs) 56 self._conn = self.connection_authenticator( 57 self.aws_access_key_id, self.aws_secret_access_key, 58 **self._kwargs) 59 return self._conn
60
61 62 -class EasyEC2(EasyAWS):
63 - def __init__(self, aws_access_key_id, aws_secret_access_key, 64 aws_ec2_path='/', aws_s3_host=None, aws_s3_path='/', 65 aws_port=None, aws_region_name=None, aws_is_secure=True, 66 aws_region_host=None, **kwargs):
67 aws_region = None 68 if aws_region_name and aws_region_host: 69 aws_region = boto.ec2.regioninfo.RegionInfo( 70 name=aws_region_name, endpoint=aws_region_host) 71 kwargs = dict(is_secure=aws_is_secure, region=aws_region, 72 port=aws_port, path=aws_ec2_path) 73 super(EasyEC2, self).__init__(aws_access_key_id, aws_secret_access_key, 74 boto.connect_ec2, **kwargs) 75 kwargs = dict(aws_s3_host=aws_s3_host, 76 aws_s3_path=aws_s3_path, 77 aws_port=aws_port, 78 aws_is_secure=aws_is_secure) 79 self.s3 = EasyS3(aws_access_key_id, aws_secret_access_key, **kwargs) 80 self._regions = None
81
82 - def __repr__(self):
83 return '<EasyEC2: %s (%s)>' % (self.region.name, self.region.endpoint)
84
85 - def connect_to_region(self, region_name):
86 """ 87 Connects to a given region if it exists, raises RegionDoesNotExist 88 otherwise. Once connected, this object will return only data from the 89 given region. 90 """ 91 region = self.get_region(region_name) 92 self._kwargs['region'] = region 93 self.reload() 94 return self
95 96 @property
97 - def region(self):
98 """ 99 Returns the current EC2 region used by this EasyEC2 object 100 """ 101 return self.conn.region
102 103 @property
104 - def regions(self):
105 """ 106 This property returns all AWS Regions, caching the results the first 107 time a request is made to Amazon 108 """ 109 if not self._regions: 110 self._regions = {} 111 regions = self.conn.get_all_regions() 112 for region in regions: 113 self._regions[region.name] = region 114 return self._regions
115
116 - def get_region(self, region_name):
117 """ 118 Returns boto Region object if it exists, raises RegionDoesNotExist 119 otherwise. 120 """ 121 if not region_name in self.regions: 122 raise exception.RegionDoesNotExist(region_name) 123 return self.regions.get(region_name)
124
125 - def list_regions(self):
126 """ 127 Print name/endpoint for all AWS regions 128 """ 129 regions = self.regions.items() 130 regions.sort(reverse=True) 131 for region in regions: 132 name, endpoint = region 133 print 'name: ', name 134 print 'endpoint: ', endpoint 135 print
136 137 @property
138 - def registered_images(self):
139 return self.conn.get_all_images(owners=["self"])
140 141 @property
142 - def executable_images(self):
143 return self.conn.get_all_images(executable_by=["self"])
144
145 - def get_registered_image(self, image_id):
146 if not image_id.startswith('ami') or len(image_id) != 12: 147 raise TypeError("invalid AMI name/id requested: %s" % image_id) 148 for img in self.registered_images: 149 if img.id == image_id: 150 return img
151
152 - def create_group(self, name, description, auth_ssh=False, 153 auth_group_traffic=False):
154 """ 155 Create security group with name/description. auth_ssh=True 156 will open port 22 to world (0.0.0.0/0). auth_group_traffic 157 will allow all traffic between instances in the same security 158 group 159 """ 160 if not name: 161 return None 162 log.info("Creating security group %s..." % name) 163 sg = self.conn.create_security_group(name, description) 164 if auth_ssh: 165 ssh_port = static.DEFAULT_SSH_PORT 166 sg.authorize('tcp', ssh_port, ssh_port, static.WORLD_CIDRIP) 167 if auth_group_traffic: 168 sg.authorize('icmp', -1, -1, 169 src_group=self.get_group_or_none(name)) 170 sg.authorize('tcp', 1, 65535, 171 src_group=self.get_group_or_none(name)) 172 sg.authorize('udp', 1, 65535, 173 src_group=self.get_group_or_none(name)) 174 return sg
175
176 - def get_all_security_groups(self, groupnames=[]):
177 """ 178 Returns all security groups 179 180 groupnames - optional list of group names to retrieve 181 """ 182 filters = {} 183 if groupnames: 184 filters = {'group-name': groupnames} 185 return self.get_security_groups(filters=filters)
186
187 - def get_group_or_none(self, name):
188 """ 189 Returns group with name if it exists otherwise returns None 190 """ 191 try: 192 return self.get_security_group(name) 193 except exception.SecurityGroupDoesNotExist: 194 pass
195
196 - def get_or_create_group(self, name, description, auth_ssh=True, 197 auth_group_traffic=False):
198 """ 199 Try to return a security group by name. If the group is not found, 200 attempt to create it. Description only applies to creation. 201 202 auth_ssh - authorize ssh traffic from world 203 auth_group_traffic - authorizes all traffic between members of the 204 group 205 """ 206 sg = self.get_group_or_none(name) 207 if not sg: 208 sg = self.create_group(name, description, auth_ssh, 209 auth_group_traffic) 210 return sg
211
212 - def get_security_group(self, groupname):
213 try: 214 return self.get_security_groups( 215 filters={'group-name': groupname})[0] 216 except boto.exception.EC2ResponseError, e: 217 if e.error_code == "InvalidGroup.NotFound": 218 raise exception.SecurityGroupDoesNotExist(groupname) 219 raise 220 except IndexError: 221 raise exception.SecurityGroupDoesNotExist(groupname)
222
223 - def get_security_groups(self, filters=None):
224 """ 225 Returns all security groups on this EC2 account 226 """ 227 return self.conn.get_all_security_groups(filters=filters)
228
229 - def get_permission_or_none(self, group, ip_protocol, from_port, to_port, 230 cidr_ip=None):
231 """ 232 Returns the rule with the specified port range permission (ip_protocol, 233 from_port, to_port, cidr_ip) defined or None if no such rule exists 234 """ 235 for rule in group.rules: 236 if rule.ip_protocol != ip_protocol: 237 continue 238 if int(rule.from_port) != from_port: 239 continue 240 if int(rule.to_port) != to_port: 241 continue 242 if cidr_ip: 243 cidr_grants = [g for g in rule.grants if g.cidr_ip == cidr_ip] 244 if not cidr_grants: 245 continue 246 return rule
247
248 - def has_permission(self, group, ip_protocol, from_port, to_port, cidr_ip):
249 """ 250 Checks whether group has the specified port range permission 251 (ip_protocol, from_port, to_port, cidr_ip) defined 252 """ 253 for rule in group.rules: 254 if rule.ip_protocol != ip_protocol: 255 continue 256 if int(rule.from_port) != from_port: 257 continue 258 if int(rule.to_port) != to_port: 259 continue 260 cidr_grants = [g for g in rule.grants if g.cidr_ip == cidr_ip] 261 if not cidr_grants: 262 continue 263 return True 264 return False
265
266 - def create_placement_group(self, name):
267 """ 268 Create a new placement group for your account. 269 This will create the placement group within the region you 270 are currently connected to. 271 """ 272 log.info("Creating placement group %s..." % name) 273 success = self.conn.create_placement_group(name) 274 if success: 275 return self.get_placement_group_or_none(name)
276
277 - def get_placement_groups(self, filters=None):
278 return self.conn.get_all_placement_groups(filters=filters)
279
280 - def get_placement_group(self, groupname=None):
281 try: 282 return self.get_placement_groups(filters={'group-name': 283 groupname})[0] 284 except boto.exception.EC2ResponseError, e: 285 if e.error_code == "InvalidPlacementGroup.Unknown": 286 raise exception.PlacementGroupDoesNotExist(groupname) 287 raise 288 except IndexError: 289 raise exception.PlacementGroupDoesNotExist(groupname)
290
291 - def get_placement_group_or_none(self, name):
292 """ 293 Returns placement group with name if it exists otherwise returns None 294 """ 295 region = self.conn.region.name 296 if not region in static.CLUSTER_REGIONS: 297 region_list = ', '.join(static.CLUSTER_REGIONS) 298 log.debug("region %s not in CLUSTER_REGIONS (%s)" % (region, 299 region_list)) 300 return 301 try: 302 return self.get_placement_group(name) 303 except exception.PlacementGroupDoesNotExist: 304 pass
305
306 - def get_or_create_placement_group(self, name):
307 """ 308 Try to return a placement group by name. 309 If the group is not found, attempt to create it. 310 """ 311 try: 312 return self.get_placement_group(name) 313 except exception.PlacementGroupDoesNotExist: 314 pg = self.create_placement_group(name) 315 return pg
316
317 - def request_instances(self, image_id, price=None, instance_type='m1.small', 318 min_count=1, max_count=1, count=1, key_name=None, 319 security_groups=None, launch_group=None, 320 availability_zone_group=None, placement=None, 321 user_data=None, placement_group=None):
322 """ 323 Convenience method for running spot or flat-rate instances 324 """ 325 if price: 326 return self.request_spot_instances( 327 price, image_id, instance_type=instance_type, 328 count=count, launch_group=launch_group, key_name=key_name, 329 security_groups=security_groups, 330 availability_zone_group=availability_zone_group, 331 placement=placement, user_data=user_data) 332 else: 333 return self.run_instances( 334 image_id, instance_type=instance_type, 335 min_count=min_count, max_count=max_count, 336 key_name=key_name, security_groups=security_groups, 337 placement=placement, user_data=user_data, 338 placement_group=placement_group)
339
340 - def request_spot_instances(self, price, image_id, instance_type='m1.small', 341 count=1, launch_group=None, key_name=None, 342 availability_zone_group=None, 343 security_groups=None, 344 placement=None, user_data=None):
345 return self.conn.request_spot_instances( 346 price, image_id, instance_type=instance_type, count=count, 347 launch_group=launch_group, key_name=key_name, 348 security_groups=security_groups, 349 availability_zone_group=availability_zone_group, 350 placement=placement, user_data=user_data)
351
352 - def run_instances(self, image_id, instance_type='m1.small', min_count=1, 353 max_count=1, key_name=None, security_groups=None, 354 placement=None, user_data=None, placement_group=None):
355 return self.conn.run_instances(image_id, instance_type=instance_type, 356 min_count=min_count, 357 max_count=max_count, 358 key_name=key_name, 359 security_groups=security_groups, 360 placement=placement, 361 user_data=user_data, 362 placement_group=placement_group)
363
364 - def create_image(self, instance_id, name, description=None, 365 no_reboot=False):
366 return self.conn.create_image(instance_id, name, 367 description=description, 368 no_reboot=no_reboot)
369
370 - def register_image(self, name, description=None, image_location=None, 371 architecture=None, kernel_id=None, ramdisk_id=None, 372 root_device_name=None, block_device_map=None):
373 return self.conn.register_image(name=name, description=description, 374 image_location=image_location, 375 architecture=architecture, 376 kernel_id=kernel_id, 377 ramdisk_id=ramdisk_id, 378 root_device_name=root_device_name, 379 block_device_map=block_device_map)
380
381 - def delete_keypair(self, name):
382 return self.conn.delete_key_pair(name)
383
384 - def create_keypair(self, name, output_file=None):
385 """ 386 Create a new EC2 keypair and optionally save to output_file 387 388 Returns boto.ec2.keypair.KeyPair 389 """ 390 if output_file: 391 output_dir = os.path.dirname(output_file) 392 if output_dir and not os.path.exists(output_dir): 393 raise exception.BaseException( 394 "output directory does not exist") 395 if os.path.exists(output_file): 396 raise exception.BaseException( 397 "cannot save keypair %s: file already exists" % \ 398 output_file) 399 kp = self.conn.create_key_pair(name) 400 if output_file: 401 try: 402 kfile = open(output_file, 'wb') 403 kfile.write(kp.material) 404 kfile.close() 405 os.chmod(output_file, 0400) 406 except IOError, e: 407 raise exception.BaseException(str(e)) 408 return kp
409
410 - def get_keypairs(self, filters={}):
411 return self.conn.get_all_key_pairs(filters=filters)
412
413 - def get_keypair(self, keypair):
414 try: 415 return self.get_keypairs(filters={'key-name': keypair})[0] 416 except boto.exception.EC2ResponseError, e: 417 if e.error_code == "InvalidKeyPair.NotFound": 418 raise exception.KeyPairDoesNotExist(keypair) 419 raise 420 except IndexError: 421 raise exception.KeyPairDoesNotExist(keypair)
422
423 - def get_keypair_or_none(self, keypair):
424 try: 425 return self.get_keypair(keypair) 426 except exception.KeyPairDoesNotExist: 427 pass
428
429 - def __print_header(self, msg):
430 print msg 431 print "-" * len(msg)
432
433 - def get_image_name(self, img):
434 image_name = re.sub('\.manifest\.xml$', '', 435 img.location.split('/')[-1]) 436 return image_name
437
438 - def get_instance_user_data(self, instance_id):
439 try: 440 attrs = self.conn.get_instance_attribute(instance_id, 'userData') 441 user_data = attrs.get('userData', '') 442 return base64.b64decode(user_data) 443 except boto.exception.EC2ResponseError, e: 444 if e.error_code == "InvalidInstanceID.NotFound": 445 raise exception.InstanceDoesNotExist(instance_id) 446 raise e
447
448 - def get_all_instances(self, instance_ids=[], filters=None):
449 reservations = self.conn.get_all_instances(instance_ids, 450 filters=filters) 451 instances = [] 452 for res in reservations: 453 insts = res.instances 454 for i in insts: 455 # set group info 456 i.groups = res.groups 457 instances.extend(insts) 458 return instances
459
460 - def get_instance(self, instance_id):
461 try: 462 return self.get_all_instances( 463 filters={'instance-id': instance_id})[0] 464 except boto.exception.EC2ResponseError, e: 465 if e.error_code == "InvalidInstanceID.NotFound": 466 raise exception.InstanceDoesNotExist(instance_id) 467 raise 468 except IndexError: 469 raise exception.InstanceDoesNotExist(instance_id)
470
471 - def is_valid_conn(self):
472 try: 473 self.get_all_instances() 474 return True 475 except boto.exception.EC2ResponseError, e: 476 cred_errs = ['AuthFailure', 'SignatureDoesNotMatch'] 477 if e.error_code in cred_errs: 478 return False 479 raise
480
481 - def get_all_spot_requests(self, spot_ids=[], filters=None):
482 spots = self.conn.get_all_spot_instance_requests(spot_ids, 483 filters=filters) 484 return spots
485
486 - def list_all_spot_instances(self, show_closed=False):
487 s = self.conn.get_all_spot_instance_requests() 488 if not s: 489 log.info("No spot instance requests found...") 490 return 491 spots = [] 492 for spot in s: 493 if spot.state in ['closed', 'cancelled'] and not show_closed: 494 continue 495 state = spot.state or 'N/A' 496 spot_id = spot.id or 'N/A' 497 spots.append(spot_id) 498 type = spot.type 499 instance_id = spot.instance_id or 'N/A' 500 create_time = spot.create_time or 'N/A' 501 launch_group = spot.launch_group or 'N/A' 502 zone_group = spot.availability_zone_group or 'N/A' 503 price = spot.price or 'N/A' 504 lspec = spot.launch_specification 505 instance_type = lspec.instance_type 506 image_id = lspec.image_id 507 zone = lspec.placement 508 groups = ', '.join([g.id for g in lspec.groups]) 509 print "id: %s" % spot_id 510 print "price: $%0.2f" % price 511 print "spot_request_type: %s" % type 512 print "state: %s" % state 513 print "instance_id: %s" % instance_id 514 print "instance_type: %s" % instance_type 515 print "image_id: %s" % image_id 516 print "zone: %s" % zone 517 print "create_time: %s" % create_time 518 print "launch_group: %s" % launch_group 519 print "zone_group: %s" % zone_group 520 print "security_groups: %s" % groups 521 print 522 if not spots: 523 log.info("No spot instance requests found...")
524
525 - def show_instance(self, instance):
526 id = instance.id or 'N/A' 527 groups = ', '.join([g.name for g in instance.groups]) 528 dns_name = instance.dns_name or 'N/A' 529 private_dns_name = instance.private_dns_name or 'N/A' 530 state = instance.state or 'N/A' 531 private_ip = instance.private_ip_address or 'N/A' 532 public_ip = instance.ip_address or 'N/A' 533 zone = instance.placement or 'N/A' 534 ami = instance.image_id or 'N/A' 535 instance_type = instance.instance_type or 'N/A' 536 keypair = instance.key_name or 'N/A' 537 uptime = utils.get_elapsed_time(instance.launch_time) or 'N/A' 538 print "id: %s" % id 539 print "dns_name: %s" % dns_name 540 print "private_dns_name: %s" % private_dns_name 541 if instance.reason: 542 print "state: %s (%s)" % (state, instance.reason) 543 else: 544 print "state: %s" % state 545 print "public_ip: %s" % public_ip 546 print "private_ip: %s" % private_ip 547 print "zone: %s" % zone 548 print "ami: %s" % ami 549 print "type: %s" % instance_type 550 print "groups: %s" % groups 551 print "keypair: %s" % keypair 552 print "uptime: %s" % uptime 553 print
554
555 - def list_all_instances(self, show_terminated=False):
556 insts = self.get_all_instances() 557 if not insts: 558 log.info("No instances found") 559 return 560 tstates = ['shutting-down', 'terminated'] 561 for instance in insts: 562 if not instance.state in tstates or show_terminated: 563 self.show_instance(instance)
564
565 - def list_images(self, images, sort_key=None, reverse=False):
566 def get_key(obj): 567 return ' '.join([obj.region.name, obj.location])
568 if not sort_key: 569 sort_key = get_key 570 imgs_i386 = [img for img in images if img.architecture == "i386"] 571 imgs_i386.sort(key=sort_key, reverse=reverse) 572 imgs_x86_64 = [img for img in images if img.architecture == "x86_64"] 573 imgs_x86_64.sort(key=sort_key, reverse=reverse) 574 print 575 self.__list_images("32bit Images:", imgs_i386) 576 self.__list_images("\n64bit Images:", imgs_x86_64) 577 print "\ntotal images: %d" % len(images) 578 print
579
580 - def list_registered_images(self):
581 images = self.registered_images 582 log.info("Your registered images:") 583 self.list_images(images)
584
585 - def list_executable_images(self):
586 images = self.executable_images 587 log.info("Private images owned by other users that you can execute:") 588 self.list_images(images)
589
590 - def __list_images(self, msg, imgs):
591 counter = 0 592 self.__print_header(msg) 593 for img in imgs: 594 name = self.get_image_name(img) 595 template = "[%d] %s %s %s" 596 if img.virtualization_type == 'hvm': 597 template += ' (HVM-EBS)' 598 elif img.root_device_type == 'ebs': 599 template += ' (EBS)' 600 print template % (counter, img.id, img.region.name, name) 601 counter += 1
602
603 - def remove_image_files(self, image_name, pretend=True):
604 if pretend: 605 log.info("Pretending to remove image files...") 606 else: 607 log.info('Removing image files...') 608 files = self.get_image_files(image_name) 609 for f in files: 610 if pretend: 611 log.info("Would remove file: %s" % f.name) 612 else: 613 log.info('Removing file %s' % f.name) 614 f.delete() 615 if not pretend: 616 files = self.get_image_files(image_name) 617 if len(files) != 0: 618 log.warn('Not all files deleted, recursing...') 619 self.remove_image_files(image_name, pretend)
620 621 @print_timing("Removing image")
622 - def remove_image(self, image_name, pretend=True, keep_image_data=True):
623 img = self.get_image(image_name) 624 if pretend: 625 log.info('Pretending to deregister AMI: %s' % img.id) 626 else: 627 log.info('Deregistering AMI: %s' % img.id) 628 img.deregister() 629 if img.root_device_type == "instance-store" and not keep_image_data: 630 self.remove_image_files(img, pretend=pretend) 631 elif img.root_device_type == "ebs" and not keep_image_data: 632 rootdevtype = img.block_device_mapping.get('/dev/sda1', None) 633 if rootdevtype: 634 snapid = rootdevtype.snapshot_id 635 if snapid: 636 snap = self.get_snapshot(snapid) 637 if pretend: 638 log.info("Would remove snapshot: %s" % snapid) 639 else: 640 log.info("Removing snapshot: %s" % snapid) 641 snap.delete()
642
643 - def list_starcluster_public_images(self):
644 images = self.conn.get_all_images(owners=[static.STARCLUSTER_OWNER_ID]) 645 log.info("Listing all public StarCluster images...") 646 imgs = [img for img in images if img.is_public] 647 648 def sc_public_sort(obj): 649 split = obj.name.split('-') 650 osname, osversion, arch = split[2:5] 651 osversion = float(osversion) 652 rc = 0 653 if split[-1].startswith('rc'): 654 rc = int(split[-1].replace('rc', '')) 655 return (osversion, rc)
656 self.list_images(imgs, sort_key=sc_public_sort, reverse=True) 657
658 - def create_volume(self, size, zone, snapshot_id=None):
659 return self.conn.create_volume(size, zone, snapshot_id)
660
661 - def remove_volume(self, volume_id):
662 vol = self.get_volume(volume_id) 663 vol.delete()
664
665 - def list_keypairs(self):
666 keypairs = self.keypairs 667 if not keypairs: 668 log.info("No keypairs found...") 669 return 670 max_length = max([len(key.name) for key in keypairs]) 671 templ = "%" + str(max_length) + "s %s" 672 for key in self.keypairs: 673 print templ % (key.name, key.fingerprint)
674
675 - def list_zones(self, region=None):
676 conn = self.conn 677 if region: 678 regs = self.conn.get_all_regions() 679 regions = [r.name for r in regs] 680 if not region in regions: 681 raise exception.RegionDoesNotExist(region) 682 for reg in regs: 683 if reg.name == region: 684 region = reg 685 break 686 kwargs = {} 687 kwargs.update(self._kwargs) 688 kwargs.update(dict(region=region)) 689 conn = self.connection_authenticator( 690 self.aws_access_key_id, self.aws_secret_access_key, **kwargs) 691 for zone in conn.get_all_zones(): 692 print 'name: ', zone.name 693 print 'region: ', zone.region.name 694 print 'status: ', zone.state 695 print
696
697 - def get_zones(self, filters=None):
698 return self.conn.get_all_zones(filters=filters)
699
700 - def get_zone(self, zone):
701 """ 702 Return zone object respresenting an EC2 availability zone 703 Raises exception.ZoneDoesNotExist if not successful 704 """ 705 try: 706 return self.get_zones(filters={'zone-name': zone})[0] 707 except boto.exception.EC2ResponseError, e: 708 if e.error_code == "InvalidZone.NotFound": 709 raise exception.ZoneDoesNotExist(zone, self.region.name) 710 except IndexError: 711 raise exception.ZoneDoesNotExist(zone, self.region.name)
712
713 - def get_zone_or_none(self, zone):
714 """ 715 Return zone object respresenting an EC2 availability zone 716 Returns None if unsuccessful 717 """ 718 try: 719 return self.get_zone(zone) 720 except exception.ZoneDoesNotExist: 721 pass
722
723 - def create_s3_image(self, instance_id, key_location, aws_user_id, 724 ec2_cert, ec2_private_key, bucket, image_name="image", 725 description=None, kernel_id=None, ramdisk_id=None, 726 remove_image_files=False, **kwargs):
727 """ 728 Create instance-store (S3) image from running instance 729 """ 730 icreator = image.S3ImageCreator(self, instance_id, key_location, 731 aws_user_id, ec2_cert, 732 ec2_private_key, bucket, 733 image_name=image_name, 734 description=description, 735 kernel_id=kernel_id, 736 ramdisk_id=ramdisk_id, 737 remove_image_files=remove_image_files) 738 return icreator.create_image()
739
740 - def create_ebs_image(self, instance_id, key_location, name, 741 description=None, snapshot_description=None, 742 kernel_id=None, ramdisk_id=None, root_vol_size=15, 743 **kwargs):
744 """ 745 Create EBS-backed image from running instance 746 """ 747 sdescription = snapshot_description 748 icreator = image.EBSImageCreator(self, instance_id, key_location, 749 name, description=description, 750 snapshot_description=sdescription, 751 kernel_id=kernel_id, 752 ramdisk_id=ramdisk_id, 753 **kwargs) 754 return icreator.create_image(size=root_vol_size)
755
756 - def get_images(self, filters=None):
757 return self.conn.get_all_images(filters=filters)
758
759 - def get_image(self, image_id):
760 """ 761 Return image object representing an AMI. 762 Raises exception.AMIDoesNotExist if unsuccessful 763 """ 764 try: 765 return self.get_images(filters={'image-id': image_id})[0] 766 except boto.exception.EC2ResponseError, e: 767 if e.error_code == "InvalidAMIID.NotFound": 768 raise exception.AMIDoesNotExist(image_id) 769 raise 770 except IndexError: 771 raise exception.AMIDoesNotExist(image_id)
772
773 - def get_image_or_none(self, image_id):
774 """ 775 Return image object representing an AMI. 776 Returns None if unsuccessful 777 """ 778 try: 779 return self.get_image(image_id) 780 except exception.AMIDoesNotExist: 781 pass
782
783 - def get_image_files(self, image):
784 """ 785 Returns a list of files on S3 for an EC2 instance-store (S3-backed) 786 image. This includes the image's manifest and part files. 787 """ 788 if not hasattr(image, 'id'): 789 image = self.get_image(image) 790 if image.root_device_type == 'ebs': 791 raise exception.AWSError( 792 "Image %s is an EBS image. No image files on S3." % image.id) 793 bucket = self.get_image_bucket(image) 794 bname = re.escape(bucket.name) 795 prefix = re.sub('^%s\/' % bname, '', image.location) 796 prefix = re.sub('\.manifest\.xml$', '', prefix) 797 files = bucket.list(prefix=prefix) 798 manifest_regex = re.compile(r'%s\.manifest\.xml' % prefix) 799 part_regex = re.compile(r'%s\.part\.(\d*)' % prefix) 800 # boto with eucalyptus returns boto.s3.prefix.Prefix class at the 801 # end of the list, we ignore these by checking for delete attr 802 files = [f for f in files if hasattr(f, 'delete') and 803 part_regex.match(f.name) or manifest_regex.match(f.name)] 804 return files
805
806 - def get_image_bucket(self, image):
807 bucket_name = image.location.split('/')[0] 808 return self.s3.get_bucket(bucket_name)
809
810 - def get_image_manifest(self, image):
811 return image.location.split('/')[-1]
812 813 @print_timing("Migrating image")
814 - def migrate_image(self, image_id, destbucket, migrate_manifest=False, 815 kernel_id=None, ramdisk_id=None, region=None, cert=None, 816 private_key=None):
817 """ 818 Migrate image_id files to destbucket 819 """ 820 if migrate_manifest: 821 utils.check_required(['ec2-migrate-manifest']) 822 if not cert: 823 raise exception.BaseException("no cert specified") 824 if not private_key: 825 raise exception.BaseException("no private_key specified") 826 if not kernel_id: 827 raise exception.BaseException("no kernel_id specified") 828 if not ramdisk_id: 829 raise exception.BaseException("no ramdisk_id specified") 830 image = self.get_image(image_id) 831 if image.root_device_type == "ebs": 832 raise exception.AWSError( 833 "The image you wish to migrate is EBS-based. " + 834 "This method only works for instance-store images") 835 files = self.get_image_files(image) 836 if not files: 837 log.info("No files found for image: %s" % image_id) 838 return 839 log.info("Migrating image: %s" % image_id) 840 widgets = [files[0].name, progressbar.Percentage(), ' ', 841 progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', 842 progressbar.ETA(), ' ', ' '] 843 counter = 0 844 num_files = len(files) 845 pbar = progressbar.ProgressBar(widgets=widgets, 846 maxval=num_files).start() 847 for f in files: 848 widgets[0] = "%s: (%s/%s)" % (f.name, counter + 1, num_files) 849 # copy file to destination bucket with the same name 850 f.copy(destbucket, f.name) 851 pbar.update(counter) 852 counter += 1 853 pbar.finish() 854 if migrate_manifest: 855 dbucket = self.s3.get_bucket(destbucket) 856 manifest_key = dbucket.get_key(self.get_image_manifest(image)) 857 f = tempfile.NamedTemporaryFile() 858 manifest_key.get_contents_to_file(f.file) 859 f.file.close() 860 cmd = ('ec2-migrate-manifest -c %s -k %s -m %s --kernel %s ' + 861 '--ramdisk %s --no-mapping ') % (cert, private_key, 862 f.name, kernel_id, 863 ramdisk_id) 864 register_cmd = "ec2-register %s/%s" % (destbucket, 865 manifest_key.name) 866 if region: 867 cmd += '--region %s' % region 868 register_cmd += " --region %s" % region 869 log.info("Migrating manifest file...") 870 retval = os.system(cmd) 871 if retval != 0: 872 raise exception.BaseException( 873 "ec2-migrate-manifest failed with status %s" % retval) 874 f.file = open(f.name, 'r') 875 manifest_key.set_contents_from_file(f.file) 876 # needed so that EC2 has permission to READ the manifest file 877 manifest_key.add_email_grant('READ', 'za-team@amazon.com') 878 f.close() 879 os.unlink(f.name + '.bak') 880 log.info("Manifest migrated successfully. You can now run:\n" + 881 register_cmd + "\nto register your migrated image.")
882
883 - def create_root_block_device_map(self, snapshot_id, 884 root_device_name='/dev/sda1', 885 add_ephemeral_drives=False, 886 ephemeral_drive_0='/dev/sdb1', 887 ephemeral_drive_1='/dev/sdc1', 888 ephemeral_drive_2='/dev/sdd1', 889 ephemeral_drive_3='/dev/sde1'):
890 """ 891 Utility method for building a new block_device_map for a given snapshot 892 id. This is useful when creating a new image from a volume snapshot. 893 The returned block device map can be used with self.register_image 894 """ 895 bmap = boto.ec2.blockdevicemapping.BlockDeviceMapping() 896 sda1 = boto.ec2.blockdevicemapping.BlockDeviceType() 897 sda1.snapshot_id = snapshot_id 898 sda1.delete_on_termination = True 899 bmap[root_device_name] = sda1 900 if add_ephemeral_drives: 901 sdb1 = boto.ec2.blockdevicemapping.BlockDeviceType() 902 sdb1.ephemeral_name = 'ephemeral0' 903 bmap[ephemeral_drive_0] = sdb1 904 sdc1 = boto.ec2.blockdevicemapping.BlockDeviceType() 905 sdc1.ephemeral_name = 'ephemeral1' 906 bmap[ephemeral_drive_1] = sdc1 907 sdd1 = boto.ec2.blockdevicemapping.BlockDeviceType() 908 sdd1.ephemeral_name = 'ephemeral2' 909 bmap[ephemeral_drive_2] = sdd1 910 sde1 = boto.ec2.blockdevicemapping.BlockDeviceType() 911 sde1.ephemeral_name = 'ephemeral3' 912 bmap[ephemeral_drive_3] = sde1 913 return bmap
914 915 @print_timing("Downloading image")
916 - def download_image_files(self, image_id, destdir):
917 """ 918 Downloads the manifest.xml and all AMI parts for image_id to destdir 919 """ 920 if not os.path.isdir(destdir): 921 raise exception.BaseException( 922 "destination directory '%s' does not exist" % destdir) 923 widgets = ['file: ', progressbar.Percentage(), ' ', 924 progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', 925 progressbar.ETA(), ' ', progressbar.FileTransferSpeed()] 926 files = self.get_image_files(image_id) 927 928 def _dl_progress_cb(trans, total): 929 pbar.update(trans)
930 log.info("Downloading image: %s" % image_id) 931 for file in files: 932 widgets[0] = "%s:" % file.name 933 pbar = progressbar.ProgressBar(widgets=widgets, 934 maxval=file.size).start() 935 file.get_contents_to_filename(os.path.join(destdir, file.name), 936 cb=_dl_progress_cb) 937 pbar.finish() 938
939 - def list_image_files(self, image_id):
940 """ 941 Print a list of files for image_id to the screen 942 """ 943 files = self.get_image_files(image_id) 944 for file in files: 945 print file.name
946 947 @property
948 - def instances(self):
949 return self.get_all_instances()
950 951 @property
952 - def keypairs(self):
953 return self.get_keypairs()
954
955 - def terminate_instances(self, instances=None):
956 if instances: 957 self.conn.terminate_instances(instances)
958
959 - def get_volumes(self, filters=None):
960 """ 961 Returns a list of all EBS volumes 962 """ 963 return self.conn.get_all_volumes(filters=filters)
964
965 - def get_volume(self, volume_id):
966 """ 967 Returns EBS volume object representing volume_id. 968 Raises exception.VolumeDoesNotExist if unsuccessful 969 """ 970 try: 971 return self.get_volumes(filters={'volume-id': volume_id})[0] 972 except boto.exception.EC2ResponseError, e: 973 if e.error_code == "InvalidVolume.NotFound": 974 raise exception.VolumeDoesNotExist(volume_id) 975 raise 976 except IndexError: 977 raise exception.VolumeDoesNotExist(volume_id)
978
979 - def get_volume_or_none(self, volume_id):
980 """ 981 Returns EBS volume object representing volume_id. 982 Returns None if unsuccessful 983 """ 984 try: 985 return self.get_volume(volume_id) 986 except exception.VolumeDoesNotExist: 987 pass
988
989 - def wait_for_snapshot(self, snapshot, refresh_interval=30):
990 snap = snapshot 991 log.info("Waiting for snapshot to complete: %s" % snap.id) 992 widgets = ['%s: ' % snap.id, '', 993 progressbar.Bar(marker=progressbar.RotatingMarker()), 994 '', progressbar.Percentage(), ' ', progressbar.ETA()] 995 pbar = progressbar.ProgressBar(widgets=widgets, maxval=100).start() 996 while snap.status != 'completed': 997 try: 998 progress = int(snap.update().replace('%', '')) 999 pbar.update(progress) 1000 except ValueError: 1001 time.sleep(5) 1002 continue 1003 time.sleep(refresh_interval) 1004 pbar.finish()
1005
1006 - def create_snapshot(self, vol, description=None, wait_for_snapshot=False, 1007 refresh_interval=30):
1008 log.info("Creating snapshot of volume: %s" % vol.id) 1009 snap = vol.create_snapshot(description) 1010 if wait_for_snapshot: 1011 self.wait_for_snapshot(snap, refresh_interval) 1012 return snap
1013
1014 - def get_snapshots(self, volume_ids=[], filters=None, owner='self'):
1015 """ 1016 Returns a list of all EBS volume snapshots 1017 """ 1018 filters = filters or {} 1019 if volume_ids: 1020 filters['volume-id'] = volume_ids 1021 return self.conn.get_all_snapshots(owner=owner, filters=filters)
1022
1023 - def get_snapshot(self, snapshot_id, owner='self'):
1024 """ 1025 Returns EBS snapshot object for snapshot_id. 1026 1027 Raises exception.SnapshotDoesNotExist if unsuccessful 1028 """ 1029 try: 1030 return self.get_snapshots(filters={'snapshot-id': snapshot_id}, 1031 owner=owner)[0] 1032 except boto.exception.EC2ResponseError, e: 1033 if e.error_code == "InvalidSnapshot.NotFound": 1034 raise exception.SnapshotDoesNotExist(snapshot_id) 1035 raise 1036 except IndexError: 1037 raise exception.SnapshotDoesNotExist(snapshot_id)
1038
1039 - def list_volumes(self, volume_id=None, status=None, attach_status=None, 1040 size=None, zone=None, snapshot_id=None, 1041 show_deleted=False, tags=None, name=None):
1042 """ 1043 Print a list of volumes to the screen 1044 """ 1045 filters = {} 1046 if status: 1047 filters['status'] = status 1048 else: 1049 filters['status'] = ['creating', 'available', 'in-use', 'error'] 1050 if show_deleted: 1051 filters['status'] += ['deleting', 'deleted'] 1052 if attach_status: 1053 filters['attachment.status'] = attach_status 1054 if volume_id: 1055 filters['volume-id'] = volume_id 1056 if size: 1057 filters['size'] = size 1058 if zone: 1059 filters['availability-zone'] = zone 1060 if snapshot_id: 1061 filters['snapshot-id'] = snapshot_id 1062 if tags: 1063 tagkeys = [] 1064 for tag in tags: 1065 val = tags.get(tag) 1066 if val: 1067 filters["tag:%s" % tag] = val 1068 elif tag: 1069 tagkeys.append(tag) 1070 if tagkeys: 1071 filters['tag-key'] = tagkeys 1072 if name: 1073 filters['tag:Name'] = name 1074 vols = self.get_volumes(filters=filters) 1075 vols.sort(key=lambda x: x.create_time) 1076 if vols: 1077 for vol in vols: 1078 print "volume_id: %s" % vol.id 1079 print "size: %sGB" % vol.size 1080 print "status: %s" % vol.status 1081 if vol.attachment_state(): 1082 print "attachment_status: %s" % vol.attachment_state() 1083 print "availability_zone: %s" % vol.zone 1084 if vol.snapshot_id: 1085 print "snapshot_id: %s" % vol.snapshot_id 1086 snapshots = self.get_snapshots(volume_ids=[vol.id]) 1087 if snapshots: 1088 snap_list = ' '.join([snap.id for snap in snapshots]) 1089 print 'snapshots: %s' % snap_list 1090 if vol.create_time: 1091 lt = utils.iso_to_localtime_tuple(vol.create_time) 1092 print "create_time: %s" % lt 1093 tags = [] 1094 for tag in vol.tags: 1095 val = vol.tags.get(tag) 1096 if val: 1097 tags.append("%s=%s" % (tag, val)) 1098 else: 1099 tags.append(tag) 1100 if tags: 1101 print "tags: %s" % ', '.join(tags) 1102 print 1103 print 'Total: %s' % len(vols)
1104
1105 - def get_spot_history(self, instance_type, start=None, end=None, plot=False, 1106 plot_server_interface="localhost", 1107 plot_launch_browser=True, plot_web_browser=None, 1108 plot_shutdown_server=True):
1109 if start and not utils.is_iso_time(start): 1110 raise exception.InvalidIsoDate(start) 1111 if end and not utils.is_iso_time(end): 1112 raise exception.InvalidIsoDate(end) 1113 pdesc = "Linux/UNIX" 1114 hist = self.conn.get_spot_price_history(start_time=start, end_time=end, 1115 instance_type=instance_type, 1116 product_description=pdesc) 1117 if not hist: 1118 raise exception.SpotHistoryError(start, end) 1119 dates = [] 1120 prices = [] 1121 data = [] 1122 for item in hist: 1123 timestamp = utils.iso_to_javascript_timestamp(item.timestamp) 1124 price = item.price 1125 dates.append(timestamp) 1126 prices.append(price) 1127 data.append([timestamp, price]) 1128 maximum = max(prices) 1129 avg = sum(prices) / float(len(prices)) 1130 log.info("Current price: $%.2f" % prices[-1]) 1131 log.info("Max price: $%.2f" % maximum) 1132 log.info("Average price: $%.2f" % avg) 1133 if plot: 1134 xaxisrange = dates[-1] - dates[0] 1135 xpanrange = [dates[0] - xaxisrange / 2., 1136 dates[-1] + xaxisrange / 2.] 1137 xzoomrange = [0.1, xpanrange[-1] - xpanrange[0]] 1138 minimum = min(prices) 1139 yaxisrange = maximum - minimum 1140 ypanrange = [minimum - yaxisrange / 2., maximum + yaxisrange / 2.] 1141 yzoomrange = [0.1, ypanrange[-1] - ypanrange[0]] 1142 context = dict(instance_type=instance_type, 1143 start=start, end=end, 1144 time_series_data=str(data), 1145 shutdown=plot_shutdown_server, 1146 xpanrange=xpanrange, ypanrange=ypanrange, 1147 xzoomrange=xzoomrange, yzoomrange=yzoomrange) 1148 log.info("", extra=dict(__raw__=True)) 1149 log.info("Starting StarCluster Webserver...") 1150 s = webtools.get_template_server('web', context=context, 1151 interface=plot_server_interface) 1152 base_url = "http://%s:%s" % s.server_address 1153 shutdown_url = '/'.join([base_url, 'shutdown']) 1154 spot_url = "http://%s:%s/spothistory.html" % s.server_address 1155 log.info("Server address is %s" % base_url) 1156 log.info("(use CTRL-C or navigate to %s to shutdown server)" % 1157 shutdown_url) 1158 if plot_launch_browser: 1159 webtools.open_browser(spot_url, plot_web_browser) 1160 else: 1161 log.info("Browse to %s to view the spot history plot" % 1162 spot_url) 1163 s.serve_forever() 1164 return data
1165
1166 - def show_console_output(self, instance_id):
1167 instance = self.get_instance(instance_id) 1168 console_output = instance.get_console_output().output 1169 print ''.join([c for c in console_output if c in string.printable])
1170
1171 1172 -class EasyS3(EasyAWS):
1173 DefaultHost = 's3.amazonaws.com' 1174 _calling_format = boto.s3.connection.OrdinaryCallingFormat() 1175
1176 - def __init__(self, aws_access_key_id, aws_secret_access_key, 1177 aws_s3_path='/', aws_port=None, aws_is_secure=True, 1178 aws_s3_host=DefaultHost, **kwargs):
1179 kwargs = dict(is_secure=aws_is_secure, 1180 host=aws_s3_host or self.DefaultHost, 1181 port=aws_port, 1182 path=aws_s3_path) 1183 if aws_s3_host: 1184 kwargs.update(dict(calling_format=self._calling_format)) 1185 super(EasyS3, self).__init__(aws_access_key_id, aws_secret_access_key, 1186 boto.connect_s3, **kwargs)
1187
1188 - def __repr__(self):
1189 return '<EasyS3: %s>' % self.conn.server_name()
1190
1191 - def create_bucket(self, bucket_name):
1192 """ 1193 Create a new bucket on S3. bucket_name must be unique, the bucket 1194 namespace is shared by all AWS users 1195 """ 1196 bucket_name = bucket_name.split('/')[0] 1197 try: 1198 return self.conn.create_bucket(bucket_name) 1199 except boto.exception.S3CreateError, e: 1200 if e.error_code == "BucketAlreadyExists": 1201 raise exception.BucketAlreadyExists(bucket_name) 1202 raise
1203
1204 - def bucket_exists(self, bucket_name):
1205 """ 1206 Check if bucket_name exists on S3 1207 """ 1208 try: 1209 return self.get_bucket(bucket_name) is not None 1210 except exception.BucketDoesNotExist: 1211 return False
1212
1213 - def get_bucket_or_none(self, bucket_name):
1214 """ 1215 Returns bucket object representing S3 bucket 1216 Returns None if unsuccessful 1217 """ 1218 try: 1219 return self.get_bucket(bucket_name) 1220 except exception.BucketDoesNotExist: 1221 pass
1222
1223 - def get_bucket(self, bucketname):
1224 """ 1225 Returns bucket object representing S3 bucket 1226 """ 1227 try: 1228 return self.conn.get_bucket(bucketname) 1229 except boto.exception.S3ResponseError, e: 1230 if e.error_code == "NoSuchBucket": 1231 raise exception.BucketDoesNotExist(bucketname) 1232 raise
1233
1234 - def list_bucket(self, bucketname):
1235 bucket = self.get_bucket(bucketname) 1236 for file in bucket.list(): 1237 if file.name: 1238 print file.name
1239
1240 - def get_buckets(self):
1241 try: 1242 buckets = self.conn.get_all_buckets() 1243 except TypeError: 1244 # hack until boto (or eucalyptus) fixes get_all_buckets 1245 raise exception.AWSError("AWS credentials are not valid") 1246 return buckets
1247
1248 - def list_buckets(self):
1249 for bucket in self.get_buckets(): 1250 print bucket.name
1251
1252 - def get_bucket_files(self, bucketname):
1253 bucket = self.get_bucket(bucketname) 1254 files = [file for file in bucket.list()] 1255 return files
1256 1257 if __name__ == "__main__": 1258 from starcluster.config import get_easy_ec2 1259 ec2 = get_easy_ec2() 1260 ec2.list_all_instances() 1261 ec2.list_registered_images() 1262