Package starcluster :: Module image
[hide private]
[frames] | no frames]

Source Code for Module starcluster.image

  1  import os 
  2  import time 
  3  import string 
  4   
  5  from starcluster import ssh 
  6  from starcluster import utils 
  7  from starcluster import exception 
  8  from starcluster.spinner import Spinner 
  9  from starcluster.utils import print_timing 
 10  from starcluster.logger import log 
11 12 13 -class ImageCreator(object):
14 """ 15 Base class for S3/EBS Image Creators. Handles fetching the host and setting 16 up a connection object as well as setting common attributes (description, 17 kernel_id, ramdisk_id) 18 19 easy_ec2 must be an awsutils.EasyEC2 object 20 21 instance_id is the id of the instance to be imaged 22 23 key_location must point to the private key file corresponding to the 24 keypair used to launch instance_id 25 """
26 - def __init__(self, easy_ec2, instance_id, key_location, description=None, 27 kernel_id=None, ramdisk_id=None):
28 self.ec2 = easy_ec2 29 self.host = self.ec2.get_instance(instance_id) 30 if self.host.state != 'running': 31 raise exception.InstanceNotRunning(self.host.id, self.host.state, 32 self.host.dns_name) 33 self.host_ssh = ssh.SSHClient(self.host.dns_name, username='root', 34 private_key=key_location) 35 self.description = description 36 self.kernel_id = kernel_id or self.host.kernel 37 self.ramdisk_id = ramdisk_id or self.host.ramdisk
38
39 - def clean_private_data(self):
40 log.info('Removing private data...') 41 conn = self.host_ssh 42 conn.execute('find /home -maxdepth 1 -type d -exec rm -rf {}/.ssh \;') 43 conn.execute('rm -f /var/log/secure') 44 conn.execute('rm -f /var/log/lastlog') 45 conn.execute('rm -rf /root/*') 46 conn.execute('rm -f ~/.bash_history') 47 conn.execute('rm -rf /tmp/*') 48 conn.execute('rm -rf /root/*.hist*') 49 conn.execute('rm -rf /var/log/*.gz')
50
51 52 -class S3ImageCreator(ImageCreator):
53 """ 54 Class for creating a new instance-store AMI from a running instance 55 """
56 - def __init__(self, easy_ec2, instance_id, key_location, aws_user_id, 57 ec2_cert, ec2_private_key, bucket, image_name='image', 58 description=None, kernel_id=None, ramdisk_id=None, 59 remove_image_files=False, **kwargs):
60 super(S3ImageCreator, self).__init__(easy_ec2, instance_id, 61 key_location, description, 62 kernel_id, ramdisk_id) 63 self.userid = aws_user_id 64 self.cert = ec2_cert 65 self.private_key = ec2_private_key 66 self.bucket = bucket 67 self.prefix = image_name 68 self.description = description 69 self.remove_image_files = remove_image_files 70 for name in self.bucket.split("/"): 71 if not utils.is_valid_bucket_name(name): 72 raise exception.InvalidBucketName(self.bucket) 73 if not utils.is_valid_image_name(self.prefix): 74 raise exception.InvalidImageName(self.prefix) 75 if not self.cert: 76 try: 77 self.cert = os.environ['EC2_CERT'] 78 except KeyError: 79 raise exception.EC2CertRequired() 80 if not self.private_key: 81 try: 82 self.private_key = os.environ['EC2_PRIVATE_KEY'] 83 except KeyError: 84 raise exception.EC2PrivateKeyRequired() 85 if not self.userid: 86 raise exception.AWSUserIdRequired() 87 if not os.path.exists(self.cert): 88 raise exception.EC2CertDoesNotExist(self.cert) 89 if not os.path.exists(self.private_key): 90 raise exception.EC2PrivateKeyDoesNotExist(self.private_key) 91 self.config_dict = { 92 'access_key': self.ec2.aws_access_key_id, 93 'secret_key': self.ec2.aws_secret_access_key, 94 'private_key': os.path.split(self.private_key)[-1], 95 'userid': self.userid, 96 'cert': os.path.split(self.cert)[-1], 97 'bucket': self.bucket, 98 'prefix': self.prefix, 99 'arch': self.host.architecture, 100 }
101
102 - def __repr__(self):
103 return "<S3ImageCreator: %s>" % self.host.id
104 105 @print_timing
106 - def create_image(self):
107 log.info("Checking for EC2 API tools...") 108 self.host_ssh.check_required(['ec2-upload-bundle', 'ec2-bundle-vol']) 109 self.ec2.s3.get_or_create_bucket(self.bucket) 110 self._remove_image_files() 111 self._bundle_image() 112 self._upload_image() 113 ami_id = self._register_image() 114 if self.remove_image_files: 115 self._remove_image_files() 116 return ami_id
117
118 - def _remove_image_files(self):
119 conn = self.host_ssh 120 conn.execute('umount /mnt/img-mnt', ignore_exit_status=True) 121 conn.execute('rm -rf /mnt/img-mnt') 122 conn.execute('rm -rf /mnt/%(prefix)s*' % self.config_dict)
123
124 - def _transfer_pem_files(self):
125 """copy pem files to /mnt on image host""" 126 conn = self.host_ssh 127 pkey_dest = "/mnt/" + os.path.basename(self.private_key) 128 cert_dest = "/mnt/" + os.path.basename(self.cert) 129 conn.put(self.private_key, pkey_dest) 130 conn.put(self.cert, cert_dest)
131 132 @print_timing
133 - def _bundle_image(self):
134 # run script to prepare the host 135 conn = self.host_ssh 136 config_dict = self.config_dict 137 self._transfer_pem_files() 138 self.clean_private_data() 139 log.info('Creating the bundled image: (please be patient)') 140 conn.execute('ec2-bundle-vol -d /mnt -k /mnt/%(private_key)s ' 141 '-c /mnt/%(cert)s -p %(prefix)s -u %(userid)s ' 142 '-r %(arch)s -e /root/.ssh' % config_dict, silent=False) 143 self._cleanup_pem_files()
144 145 @print_timing
146 - def _upload_image(self):
147 log.info('Uploading bundled image: (please be patient)') 148 conn = self.host_ssh 149 config_dict = self.config_dict 150 conn.execute('ec2-upload-bundle -b %(bucket)s ' 151 '-m /mnt/%(prefix)s.manifest.xml -a %(access_key)s ' 152 '-s %(secret_key)s' % config_dict, silent=False)
153
154 - def _cleanup(self):
155 #just in case... 156 self._cleanup_pem_files() 157 conn = self.host_ssh 158 conn.execute('rm -f ~/.bash_history', silent=False)
159
160 - def _cleanup_pem_files(self):
161 log.info('Cleaning up...') 162 # delete keys and remove bash history 163 conn = self.host_ssh 164 conn.execute('rm -f /mnt/*.pem /mnt/*.pem', silent=False)
165
166 - def _register_image(self):
167 # register image in s3 with ec2 168 conn = self.ec2 169 config_dict = self.config_dict 170 return conn.register_image( 171 self.prefix, 172 description=self.description, 173 image_location="%(bucket)s/%(prefix)s.manifest.xml" % config_dict, 174 kernel_id=self.kernel_id, 175 ramdisk_id=self.ramdisk_id, 176 architecture=config_dict.get('arch'), 177 )
178
179 180 -class EBSImageCreator(ImageCreator):
181 """ 182 Creates a new EBS image from a running instance 183 184 If the instance is an instance-store image, then this class will create a 185 new volume, attach it to the instance, sync the root filesystem to the 186 volume, detach the volume, snapshot it, and then create a new AMI from the 187 snapshot 188 189 If the instance is EBS-backed, this class simply calls ec2.create_image 190 which tells Amazon to create a new image in a single API call. 191 """ 192
193 - def __init__(self, easy_ec2, instance_id, key_location, name, 194 description=None, snapshot_description=None, 195 kernel_id=None, ramdisk_id=None, **kwargs):
196 super(EBSImageCreator, self).__init__(easy_ec2, instance_id, 197 key_location, description, 198 kernel_id, ramdisk_id) 199 self.name = name 200 self.description = description 201 self.snapshot_description = snapshot_description or description 202 self._snap = None 203 self._vol = None
204 205 @print_timing
206 - def create_image(self, size=15):
207 try: 208 self.clean_private_data() 209 if self.host.root_device_type == "ebs": 210 return self._create_image_from_ebs(size) 211 return self._create_image_from_instance_store(size) 212 except: 213 log.error("Error occurred while creating image") 214 if self._snap: 215 log.error("Removing generated snapshot '%s'" % self._snap) 216 self._snap.delete() 217 if self._vol: 218 log.error("Removing generated volume '%s'" % self._vol.id) 219 self._vol.detach(force=True) 220 self._vol.delete() 221 raise
222
223 - def _create_image_from_ebs(self, size=15):
224 log.info("Creating EBS image...") 225 imgid = self.ec2.create_image(self.host.id, self.name, 226 self.description) 227 log.info("Waiting for AMI %s to become available..." % imgid, 228 extra=dict(__nonewline__=True)) 229 img = self.ec2.get_image(imgid) 230 s = Spinner() 231 s.start() 232 while img.state == "pending": 233 time.sleep(15) 234 if img.update() == "failed": 235 raise exception.AWSError( 236 "EBS image creation failed for AMI %s" % imgid) 237 s.stop() 238 return imgid
239
240 - def _create_image_from_instance_store(self, size=15):
241 host = self.host 242 host_ssh = self.host_ssh 243 log.info("Creating new EBS-backed image from instance-store instance") 244 log.info("Creating new root volume...") 245 vol = self._vol = self.ec2.create_volume(size, host.placement) 246 log.info("Created new volume: %s" % vol.id) 247 while vol.update() != 'available': 248 time.sleep(5) 249 dev = None 250 for i in string.ascii_lowercase[::-1]: 251 dev = '/dev/sd%s' % i 252 if not dev in host.block_device_mapping: 253 break 254 log.info("Attaching volume %s to instance %s on %s" % 255 (vol.id, host.id, dev)) 256 vol.attach(host.id, dev) 257 while vol.update() != 'in-use': 258 time.sleep(5) 259 while not host_ssh.path_exists(dev): 260 time.sleep(5) 261 host_ssh.execute('mkfs.ext3 -F %s' % dev) 262 mount_point = '/ebs' 263 while host_ssh.path_exists(mount_point): 264 mount_point += '1' 265 host_ssh.mkdir(mount_point) 266 log.info("Mounting %s on %s" % (dev, mount_point)) 267 host_ssh.execute('mount %s %s' % (dev, mount_point)) 268 log.info("Configuring /etc/fstab") 269 host_ssh.remove_lines_from_file('/etc/fstab', '/mnt') 270 fstab = host_ssh.remote_file('/etc/fstab', 'a') 271 fstab.write('/dev/sdb1 /mnt auto defaults,nobootwait 0 0\n') 272 fstab.close() 273 log.info("Syncing root filesystem to new volume (%s)" % vol.id) 274 host_ssh.execute( 275 'rsync -avx --exclude %(mpt)s --exclude /root/.ssh / %(mpt)s' % 276 {'mpt': mount_point}) 277 log.info("Unmounting %s from %s" % (dev, mount_point)) 278 host_ssh.execute('umount %s' % mount_point) 279 log.info("Detaching volume %s from %s" % (dev, mount_point)) 280 vol.detach() 281 while vol.update() != 'available': 282 time.sleep(5) 283 sdesc = self.snapshot_description 284 snap = self._snap = self.ec2.create_snapshot(vol, 285 description=sdesc, 286 wait_for_snapshot=True) 287 log.info("New snapshot created: %s" % snap.id) 288 log.info("Removing generated volume %s" % vol.id) 289 vol.delete() 290 log.info("Creating root block device map using snapshot %s" % snap.id) 291 bmap = self.ec2.create_root_block_device_map(snap.id, 292 add_ephemeral_drives=True) 293 log.info("Registering new image...") 294 img_id = self.ec2.register_image(name=self.name, 295 description=self.description, 296 architecture=host.architecture, 297 kernel_id=self.kernel_id, 298 ramdisk_id=self.ramdisk_id, 299 root_device_name='/dev/sda1', 300 block_device_map=bmap) 301 return img_id
302 303 304 # for backwards compatibility 305 EC2ImageCreator = S3ImageCreator 306