Package starcluster :: Module image
[hide private]
[frames] | no frames]

Source Code for Module starcluster.image

  1  #!/usr/bin/env python 
  2  import os 
  3  import time 
  4  import string 
  5   
  6  from starcluster import ssh 
  7  from starcluster import utils 
  8  from starcluster import exception 
  9  from starcluster.spinner import Spinner 
 10  from starcluster.utils import print_timing 
 11  from starcluster.logger import log 
12 13 14 -class ImageCreator(object):
15 """ 16 Base class for S3/EBS Image Creators. Handles fetching the host and setting 17 up a connection object as well as setting common attributes (description, 18 kernel_id, ramdisk_id) 19 20 easy_ec2 must be an awsutils.EasyEC2 object 21 22 instance_id is the id of the instance to be imaged 23 24 key_location must point to the private key file corresponding to the 25 keypair used to launch instance_id 26 """
27 - def __init__(self, easy_ec2, instance_id, key_location, description=None, 28 kernel_id=None, ramdisk_id=None):
29 self.ec2 = easy_ec2 30 self.host = self.ec2.get_instance(instance_id) 31 if self.host.state != 'running': 32 raise exception.InstanceNotRunning(self.host.id, self.host.state, 33 self.host.dns_name) 34 self.host_ssh = ssh.SSHClient(self.host.dns_name, username='root', 35 private_key=key_location) 36 self.description = description 37 self.kernel_id = kernel_id or self.host.kernel 38 self.ramdisk_id = ramdisk_id or self.host.ramdisk
39
40 - def clean_private_data(self):
41 log.info('Removing private data...') 42 conn = self.host_ssh 43 conn.execute('find /home -maxdepth 1 -type d -exec rm -rf {}/.ssh \;') 44 conn.execute('rm -f /var/log/secure') 45 conn.execute('rm -f /var/log/lastlog') 46 conn.execute('rm -rf /root/*') 47 conn.execute('rm -f ~/.bash_history') 48 conn.execute('rm -rf /tmp/*') 49 conn.execute('rm -rf /root/*.hist*') 50 conn.execute('rm -rf /var/log/*.gz')
51
52 53 -class S3ImageCreator(ImageCreator):
54 """ 55 Class for creating a new instance-store AMI from a running instance 56 """
57 - def __init__(self, easy_ec2, instance_id, key_location, aws_user_id, 58 ec2_cert, ec2_private_key, bucket, image_name='image', 59 description=None, kernel_id=None, ramdisk_id=None, 60 remove_image_files=False, **kwargs):
61 super(S3ImageCreator, self).__init__(easy_ec2, instance_id, 62 key_location, description, 63 kernel_id, ramdisk_id) 64 self.userid = aws_user_id 65 self.cert = ec2_cert 66 self.private_key = ec2_private_key 67 self.bucket = bucket 68 self.prefix = image_name 69 self.description = description 70 self.remove_image_files = remove_image_files 71 for name in self.bucket.split("/"): 72 if not utils.is_valid_bucket_name(name): 73 raise exception.InvalidBucketName(self.bucket) 74 if not utils.is_valid_image_name(self.prefix): 75 raise exception.InvalidImageName(self.prefix) 76 if not self.cert: 77 try: 78 self.cert = os.environ['EC2_CERT'] 79 except KeyError: 80 raise exception.EC2CertRequired() 81 if not self.private_key: 82 try: 83 self.private_key = os.environ['EC2_PRIVATE_KEY'] 84 except KeyError: 85 raise exception.EC2PrivateKeyRequired() 86 if not self.userid: 87 raise exception.AWSUserIdRequired() 88 if not os.path.exists(self.cert): 89 raise exception.EC2CertDoesNotExist(self.cert) 90 if not os.path.exists(self.private_key): 91 raise exception.EC2PrivateKeyDoesNotExist(self.private_key) 92 self.config_dict = { 93 'access_key': self.ec2.aws_access_key_id, 94 'secret_key': self.ec2.aws_secret_access_key, 95 'private_key': os.path.split(self.private_key)[-1], 96 'userid': self.userid, 97 'cert': os.path.split(self.cert)[-1], 98 'bucket': self.bucket, 99 'prefix': self.prefix, 100 'arch': self.host.architecture, 101 }
102
103 - def __repr__(self):
104 return "<S3ImageCreator: %s>" % self.host.id
105 106 @print_timing
107 - def create_image(self):
108 log.info("Creating bucket %s" % self.bucket) 109 self.ec2.s3.create_bucket(self.bucket) 110 log.info("Checking for EC2 API tools...") 111 self.host_ssh.check_required(['ec2-upload-bundle', 'ec2-bundle-vol']) 112 self._remove_image_files() 113 self._bundle_image() 114 self._upload_image() 115 ami_id = self._register_image() 116 if self.remove_image_files: 117 self._remove_image_files() 118 return ami_id
119
120 - def _remove_image_files(self):
121 conn = self.host_ssh 122 conn.execute('umount /mnt/img-mnt', ignore_exit_status=True) 123 conn.execute('rm -rf /mnt/img-mnt') 124 conn.execute('rm -rf /mnt/%(prefix)s*' % self.config_dict)
125
126 - def _transfer_pem_files(self):
127 """copy pem files to /mnt on image host""" 128 conn = self.host_ssh 129 pkey_dest = "/mnt/" + os.path.basename(self.private_key) 130 cert_dest = "/mnt/" + os.path.basename(self.cert) 131 conn.put(self.private_key, pkey_dest) 132 conn.put(self.cert, cert_dest)
133 134 @print_timing
135 - def _bundle_image(self):
136 # run script to prepare the host 137 conn = self.host_ssh 138 config_dict = self.config_dict 139 self._transfer_pem_files() 140 self.clean_private_data() 141 log.info('Creating the bundled image: (please be patient)') 142 conn.execute('ec2-bundle-vol -d /mnt -k /mnt/%(private_key)s \ 143 -c /mnt/%(cert)s -p %(prefix)s -u %(userid)s -r %(arch)s -e /root/.ssh' % \ 144 config_dict, 145 silent=False) 146 self._cleanup_pem_files()
147 148 @print_timing
149 - def _upload_image(self):
150 log.info('Uploading bundled image: (please be patient)') 151 conn = self.host_ssh 152 config_dict = self.config_dict 153 conn.execute('ec2-upload-bundle -b %(bucket)s \ 154 -m /mnt/%(prefix)s.manifest.xml -a %(access_key)s -s %(secret_key)s' % \ 155 config_dict, silent=False)
156
157 - def _cleanup(self):
158 #just in case... 159 self._cleanup_pem_files() 160 conn = self.host_ssh 161 conn.execute('rm -f ~/.bash_history', silent=False)
162
163 - def _cleanup_pem_files(self):
164 log.info('Cleaning up...') 165 # delete keys and remove bash history 166 conn = self.host_ssh 167 conn.execute('rm -f /mnt/*.pem /mnt/*.pem', silent=False)
168
169 - def _register_image(self):
170 # register image in s3 with ec2 171 conn = self.ec2 172 config_dict = self.config_dict 173 return conn.register_image( 174 self.prefix, 175 description=self.description, 176 image_location="%(bucket)s/%(prefix)s.manifest.xml" % config_dict, 177 kernel_id=self.kernel_id, 178 ramdisk_id=self.ramdisk_id, 179 architecture=config_dict.get('arch'), 180 )
181
182 183 -class EBSImageCreator(ImageCreator):
184 """ 185 Creates a new EBS image from a running instance 186 187 If the instance is an instance-store image, then this class will create a 188 new volume, attach it to the instance, sync the root filesystem to the 189 volume, detach the volume, snapshot it, and then create a new AMI from the 190 snapshot 191 192 If the instance is EBS-backed, this class simply calls ec2.create_image 193 which tells Amazon to create a new image in a single API call. 194 """ 195
196 - def __init__(self, easy_ec2, instance_id, key_location, name, 197 description=None, snapshot_description=None, 198 kernel_id=None, ramdisk_id=None, **kwargs):
199 super(EBSImageCreator, self).__init__(easy_ec2, instance_id, 200 key_location, description, 201 kernel_id, ramdisk_id) 202 self.name = name 203 self.description = description 204 self.snapshot_description = snapshot_description or description 205 self._snap = None 206 self._vol = None
207 208 @print_timing
209 - def create_image(self, size=15):
210 try: 211 self.clean_private_data() 212 if self.host.root_device_type == "ebs": 213 return self._create_image_from_ebs(size) 214 return self._create_image_from_instance_store(size) 215 except: 216 log.error("Error occured while creating image") 217 if self._snap: 218 log.error("Removing generated snapshot '%s'" % self._snap) 219 self._snap.delete() 220 if self._vol: 221 log.error("Removing generated volume '%s'" % self._vol.id) 222 self._vol.detach(force=True) 223 self._vol.delete() 224 raise
225
226 - def _create_image_from_ebs(self, size=15):
227 log.info("Creating EBS image...") 228 imgid = self.ec2.create_image(self.host.id, self.name, 229 self.description) 230 log.info("Waiting for AMI %s to become available..." % imgid, 231 extra=dict(__nonewline__=True)) 232 img = self.ec2.get_image(imgid) 233 s = Spinner() 234 s.start() 235 while img.state == "pending": 236 time.sleep(15) 237 if img.update() == "failed": 238 raise exception.AWSError( 239 "EBS image creation failed for AMI %s" % imgid) 240 s.stop() 241 return imgid
242
243 - def _create_image_from_instance_store(self, size=15):
244 host = self.host 245 host_ssh = self.host_ssh 246 log.info("Creating new EBS-backed image from instance-store instance") 247 log.info("Creating new root volume...") 248 vol = self._vol = self.ec2.create_volume(size, host.placement) 249 log.info("Created new volume: %s" % vol.id) 250 while vol.update() != 'available': 251 time.sleep(5) 252 dev = None 253 for i in string.ascii_lowercase[::-1]: 254 dev = '/dev/sd%s' % i 255 if not dev in host.block_device_mapping: 256 break 257 log.info("Attaching volume %s to instance %s on %s" % 258 (vol.id, host.id, dev)) 259 vol.attach(host.id, dev) 260 while vol.update() != 'in-use': 261 time.sleep(5) 262 while not host_ssh.path_exists(dev): 263 time.sleep(5) 264 host_ssh.execute('mkfs.ext3 -F %s' % dev) 265 mount_point = '/ebs' 266 while host_ssh.path_exists(mount_point): 267 mount_point += '1' 268 host_ssh.mkdir(mount_point) 269 log.info("Mounting %s on %s" % (dev, mount_point)) 270 host_ssh.execute('mount %s %s' % (dev, mount_point)) 271 log.info("Configuring /etc/fstab") 272 host_ssh.remove_lines_from_file('/etc/fstab', '/mnt') 273 fstab = host_ssh.remote_file('/etc/fstab', 'a') 274 fstab.write('/dev/sdb1 /mnt auto defaults,nobootwait 0 0\n') 275 fstab.close() 276 log.info("Syncing root filesystem to new volume (%s)" % vol.id) 277 host_ssh.execute( 278 'rsync -avx --exclude %(mpt)s --exclude /root/.ssh / %(mpt)s' % 279 {'mpt': mount_point}) 280 log.info("Unmounting %s from %s" % (dev, mount_point)) 281 host_ssh.execute('umount %s' % mount_point) 282 log.info("Detaching volume %s from %s" % (dev, mount_point)) 283 vol.detach() 284 while vol.update() != 'available': 285 time.sleep(5) 286 sdesc = self.snapshot_description 287 snap = self._snap = self.ec2.create_snapshot(vol, 288 description=sdesc, 289 wait_for_snapshot=True) 290 log.info("New snapshot created: %s" % snap.id) 291 log.info("Removing generated volume %s" % vol.id) 292 vol.delete() 293 log.info("Creating root block device map using snapshot %s" % snap.id) 294 bmap = self.ec2.create_root_block_device_map(snap.id, 295 add_ephemeral_drives=True) 296 log.info("Registering new image...") 297 img_id = self.ec2.register_image(name=self.name, 298 description=self.description, 299 architecture=host.architecture, 300 kernel_id=self.kernel_id, 301 ramdisk_id=self.ramdisk_id, 302 root_device_name='/dev/sda1', 303 block_device_map=bmap) 304 return img_id
305 306 307 # for backwards compatibility 308 EC2ImageCreator = S3ImageCreator 309