1
2 import os
3 import time
4 import string
5
6 from starcluster import ssh
7 from starcluster import utils
8 from starcluster import exception
9 from starcluster.spinner import Spinner
10 from starcluster.utils import print_timing
11 from starcluster.logger import log
15 """
16 Base class for S3/EBS Image Creators. Handles fetching the host and setting
17 up a connection object as well as setting common attributes (description,
18 kernel_id, ramdisk_id)
19
20 easy_ec2 must be an awsutils.EasyEC2 object
21
22 instance_id is the id of the instance to be imaged
23
24 key_location must point to the private key file corresponding to the
25 keypair used to launch instance_id
26 """
27 - def __init__(self, easy_ec2, instance_id, key_location, description=None,
28 kernel_id=None, ramdisk_id=None):
39
51
54 """
55 Class for creating a new instance-store AMI from a running instance
56 """
57 - def __init__(self, easy_ec2, instance_id, key_location, aws_user_id,
58 ec2_cert, ec2_private_key, bucket, image_name='image',
59 description=None, kernel_id=None, ramdisk_id=None,
60 remove_image_files=False, **kwargs):
61 super(S3ImageCreator, self).__init__(easy_ec2, instance_id,
62 key_location, description,
63 kernel_id, ramdisk_id)
64 self.userid = aws_user_id
65 self.cert = ec2_cert
66 self.private_key = ec2_private_key
67 self.bucket = bucket
68 self.prefix = image_name
69 self.description = description
70 self.remove_image_files = remove_image_files
71 for name in self.bucket.split("/"):
72 if not utils.is_valid_bucket_name(name):
73 raise exception.InvalidBucketName(self.bucket)
74 if not utils.is_valid_image_name(self.prefix):
75 raise exception.InvalidImageName(self.prefix)
76 if not self.cert:
77 try:
78 self.cert = os.environ['EC2_CERT']
79 except KeyError:
80 raise exception.EC2CertRequired()
81 if not self.private_key:
82 try:
83 self.private_key = os.environ['EC2_PRIVATE_KEY']
84 except KeyError:
85 raise exception.EC2PrivateKeyRequired()
86 if not self.userid:
87 raise exception.AWSUserIdRequired()
88 if not os.path.exists(self.cert):
89 raise exception.EC2CertDoesNotExist(self.cert)
90 if not os.path.exists(self.private_key):
91 raise exception.EC2PrivateKeyDoesNotExist(self.private_key)
92 self.config_dict = {
93 'access_key': self.ec2.aws_access_key_id,
94 'secret_key': self.ec2.aws_secret_access_key,
95 'private_key': os.path.split(self.private_key)[-1],
96 'userid': self.userid,
97 'cert': os.path.split(self.cert)[-1],
98 'bucket': self.bucket,
99 'prefix': self.prefix,
100 'arch': self.host.architecture,
101 }
102
104 return "<S3ImageCreator: %s>" % self.host.id
105
106 @print_timing
119
121 conn = self.host_ssh
122 conn.execute('umount /mnt/img-mnt', ignore_exit_status=True)
123 conn.execute('rm -rf /mnt/img-mnt')
124 conn.execute('rm -rf /mnt/%(prefix)s*' % self.config_dict)
125
127 """copy pem files to /mnt on image host"""
128 conn = self.host_ssh
129 pkey_dest = "/mnt/" + os.path.basename(self.private_key)
130 cert_dest = "/mnt/" + os.path.basename(self.cert)
131 conn.put(self.private_key, pkey_dest)
132 conn.put(self.cert, cert_dest)
133
134 @print_timing
136
137 conn = self.host_ssh
138 config_dict = self.config_dict
139 self._transfer_pem_files()
140 self.clean_private_data()
141 log.info('Creating the bundled image: (please be patient)')
142 conn.execute('ec2-bundle-vol -d /mnt -k /mnt/%(private_key)s \
143 -c /mnt/%(cert)s -p %(prefix)s -u %(userid)s -r %(arch)s -e /root/.ssh' % \
144 config_dict,
145 silent=False)
146 self._cleanup_pem_files()
147
148 @print_timing
150 log.info('Uploading bundled image: (please be patient)')
151 conn = self.host_ssh
152 config_dict = self.config_dict
153 conn.execute('ec2-upload-bundle -b %(bucket)s \
154 -m /mnt/%(prefix)s.manifest.xml -a %(access_key)s -s %(secret_key)s' % \
155 config_dict, silent=False)
156
162
164 log.info('Cleaning up...')
165
166 conn = self.host_ssh
167 conn.execute('rm -f /mnt/*.pem /mnt/*.pem', silent=False)
168
170
171 conn = self.ec2
172 config_dict = self.config_dict
173 return conn.register_image(
174 self.prefix,
175 description=self.description,
176 image_location="%(bucket)s/%(prefix)s.manifest.xml" % config_dict,
177 kernel_id=self.kernel_id,
178 ramdisk_id=self.ramdisk_id,
179 architecture=config_dict.get('arch'),
180 )
181
184 """
185 Creates a new EBS image from a running instance
186
187 If the instance is an instance-store image, then this class will create a
188 new volume, attach it to the instance, sync the root filesystem to the
189 volume, detach the volume, snapshot it, and then create a new AMI from the
190 snapshot
191
192 If the instance is EBS-backed, this class simply calls ec2.create_image
193 which tells Amazon to create a new image in a single API call.
194 """
195
196 - def __init__(self, easy_ec2, instance_id, key_location, name,
197 description=None, snapshot_description=None,
198 kernel_id=None, ramdisk_id=None, **kwargs):
199 super(EBSImageCreator, self).__init__(easy_ec2, instance_id,
200 key_location, description,
201 kernel_id, ramdisk_id)
202 self.name = name
203 self.description = description
204 self.snapshot_description = snapshot_description or description
205 self._snap = None
206 self._vol = None
207
208 @print_timing
210 try:
211 self.clean_private_data()
212 if self.host.root_device_type == "ebs":
213 return self._create_image_from_ebs(size)
214 return self._create_image_from_instance_store(size)
215 except:
216 log.error("Error occured while creating image")
217 if self._snap:
218 log.error("Removing generated snapshot '%s'" % self._snap)
219 self._snap.delete()
220 if self._vol:
221 log.error("Removing generated volume '%s'" % self._vol.id)
222 self._vol.detach(force=True)
223 self._vol.delete()
224 raise
225
227 log.info("Creating EBS image...")
228 imgid = self.ec2.create_image(self.host.id, self.name,
229 self.description)
230 log.info("Waiting for AMI %s to become available..." % imgid,
231 extra=dict(__nonewline__=True))
232 img = self.ec2.get_image(imgid)
233 s = Spinner()
234 s.start()
235 while img.state == "pending":
236 time.sleep(15)
237 if img.update() == "failed":
238 raise exception.AWSError(
239 "EBS image creation failed for AMI %s" % imgid)
240 s.stop()
241 return imgid
242
244 host = self.host
245 host_ssh = self.host_ssh
246 log.info("Creating new EBS-backed image from instance-store instance")
247 log.info("Creating new root volume...")
248 vol = self._vol = self.ec2.create_volume(size, host.placement)
249 log.info("Created new volume: %s" % vol.id)
250 while vol.update() != 'available':
251 time.sleep(5)
252 dev = None
253 for i in string.ascii_lowercase[::-1]:
254 dev = '/dev/sd%s' % i
255 if not dev in host.block_device_mapping:
256 break
257 log.info("Attaching volume %s to instance %s on %s" %
258 (vol.id, host.id, dev))
259 vol.attach(host.id, dev)
260 while vol.update() != 'in-use':
261 time.sleep(5)
262 while not host_ssh.path_exists(dev):
263 time.sleep(5)
264 host_ssh.execute('mkfs.ext3 -F %s' % dev)
265 mount_point = '/ebs'
266 while host_ssh.path_exists(mount_point):
267 mount_point += '1'
268 host_ssh.mkdir(mount_point)
269 log.info("Mounting %s on %s" % (dev, mount_point))
270 host_ssh.execute('mount %s %s' % (dev, mount_point))
271 log.info("Configuring /etc/fstab")
272 host_ssh.remove_lines_from_file('/etc/fstab', '/mnt')
273 fstab = host_ssh.remote_file('/etc/fstab', 'a')
274 fstab.write('/dev/sdb1 /mnt auto defaults,nobootwait 0 0\n')
275 fstab.close()
276 log.info("Syncing root filesystem to new volume (%s)" % vol.id)
277 host_ssh.execute(
278 'rsync -avx --exclude %(mpt)s --exclude /root/.ssh / %(mpt)s' %
279 {'mpt': mount_point})
280 log.info("Unmounting %s from %s" % (dev, mount_point))
281 host_ssh.execute('umount %s' % mount_point)
282 log.info("Detaching volume %s from %s" % (dev, mount_point))
283 vol.detach()
284 while vol.update() != 'available':
285 time.sleep(5)
286 sdesc = self.snapshot_description
287 snap = self._snap = self.ec2.create_snapshot(vol,
288 description=sdesc,
289 wait_for_snapshot=True)
290 log.info("New snapshot created: %s" % snap.id)
291 log.info("Removing generated volume %s" % vol.id)
292 vol.delete()
293 log.info("Creating root block device map using snapshot %s" % snap.id)
294 bmap = self.ec2.create_root_block_device_map(snap.id,
295 add_ephemeral_drives=True)
296 log.info("Registering new image...")
297 img_id = self.ec2.register_image(name=self.name,
298 description=self.description,
299 architecture=host.architecture,
300 kernel_id=self.kernel_id,
301 ramdisk_id=self.ramdisk_id,
302 root_device_name='/dev/sda1',
303 block_device_map=bmap)
304 return img_id
305
306
307
308 EC2ImageCreator = S3ImageCreator
309