1 import os
2 import time
3 import string
4
5 from starcluster import ssh
6 from starcluster import utils
7 from starcluster import exception
8 from starcluster.spinner import Spinner
9 from starcluster.utils import print_timing
10 from starcluster.logger import log
14 """
15 Base class for S3/EBS Image Creators. Handles fetching the host and setting
16 up a connection object as well as setting common attributes (description,
17 kernel_id, ramdisk_id)
18
19 easy_ec2 must be an awsutils.EasyEC2 object
20
21 instance_id is the id of the instance to be imaged
22
23 key_location must point to the private key file corresponding to the
24 keypair used to launch instance_id
25 """
26 - def __init__(self, easy_ec2, instance_id, key_location, description=None,
27 kernel_id=None, ramdisk_id=None):
38
50
53 """
54 Class for creating a new instance-store AMI from a running instance
55 """
56 - def __init__(self, easy_ec2, instance_id, key_location, aws_user_id,
57 ec2_cert, ec2_private_key, bucket, image_name='image',
58 description=None, kernel_id=None, ramdisk_id=None,
59 remove_image_files=False, **kwargs):
60 super(S3ImageCreator, self).__init__(easy_ec2, instance_id,
61 key_location, description,
62 kernel_id, ramdisk_id)
63 self.userid = aws_user_id
64 self.cert = ec2_cert
65 self.private_key = ec2_private_key
66 self.bucket = bucket
67 self.prefix = image_name
68 self.description = description
69 self.remove_image_files = remove_image_files
70 for name in self.bucket.split("/"):
71 if not utils.is_valid_bucket_name(name):
72 raise exception.InvalidBucketName(self.bucket)
73 if not utils.is_valid_image_name(self.prefix):
74 raise exception.InvalidImageName(self.prefix)
75 if not self.cert:
76 try:
77 self.cert = os.environ['EC2_CERT']
78 except KeyError:
79 raise exception.EC2CertRequired()
80 if not self.private_key:
81 try:
82 self.private_key = os.environ['EC2_PRIVATE_KEY']
83 except KeyError:
84 raise exception.EC2PrivateKeyRequired()
85 if not self.userid:
86 raise exception.AWSUserIdRequired()
87 if not os.path.exists(self.cert):
88 raise exception.EC2CertDoesNotExist(self.cert)
89 if not os.path.exists(self.private_key):
90 raise exception.EC2PrivateKeyDoesNotExist(self.private_key)
91 self.config_dict = {
92 'access_key': self.ec2.aws_access_key_id,
93 'secret_key': self.ec2.aws_secret_access_key,
94 'private_key': os.path.split(self.private_key)[-1],
95 'userid': self.userid,
96 'cert': os.path.split(self.cert)[-1],
97 'bucket': self.bucket,
98 'prefix': self.prefix,
99 'arch': self.host.architecture,
100 }
101
103 return "<S3ImageCreator: %s>" % self.host.id
104
105 @print_timing
117
119 conn = self.host_ssh
120 conn.execute('umount /mnt/img-mnt', ignore_exit_status=True)
121 conn.execute('rm -rf /mnt/img-mnt')
122 conn.execute('rm -rf /mnt/%(prefix)s*' % self.config_dict)
123
125 """copy pem files to /mnt on image host"""
126 conn = self.host_ssh
127 pkey_dest = "/mnt/" + os.path.basename(self.private_key)
128 cert_dest = "/mnt/" + os.path.basename(self.cert)
129 conn.put(self.private_key, pkey_dest)
130 conn.put(self.cert, cert_dest)
131
132 @print_timing
134
135 conn = self.host_ssh
136 config_dict = self.config_dict
137 self._transfer_pem_files()
138 self.clean_private_data()
139 log.info('Creating the bundled image: (please be patient)')
140 conn.execute('ec2-bundle-vol -d /mnt -k /mnt/%(private_key)s '
141 '-c /mnt/%(cert)s -p %(prefix)s -u %(userid)s '
142 '-r %(arch)s -e /root/.ssh' % config_dict, silent=False)
143 self._cleanup_pem_files()
144
145 @print_timing
147 log.info('Uploading bundled image: (please be patient)')
148 conn = self.host_ssh
149 config_dict = self.config_dict
150 conn.execute('ec2-upload-bundle -b %(bucket)s '
151 '-m /mnt/%(prefix)s.manifest.xml -a %(access_key)s '
152 '-s %(secret_key)s' % config_dict, silent=False)
153
159
161 log.info('Cleaning up...')
162
163 conn = self.host_ssh
164 conn.execute('rm -f /mnt/*.pem /mnt/*.pem', silent=False)
165
167
168 conn = self.ec2
169 config_dict = self.config_dict
170 return conn.register_image(
171 self.prefix,
172 description=self.description,
173 image_location="%(bucket)s/%(prefix)s.manifest.xml" % config_dict,
174 kernel_id=self.kernel_id,
175 ramdisk_id=self.ramdisk_id,
176 architecture=config_dict.get('arch'),
177 )
178
181 """
182 Creates a new EBS image from a running instance
183
184 If the instance is an instance-store image, then this class will create a
185 new volume, attach it to the instance, sync the root filesystem to the
186 volume, detach the volume, snapshot it, and then create a new AMI from the
187 snapshot
188
189 If the instance is EBS-backed, this class simply calls ec2.create_image
190 which tells Amazon to create a new image in a single API call.
191 """
192
193 - def __init__(self, easy_ec2, instance_id, key_location, name,
194 description=None, snapshot_description=None,
195 kernel_id=None, ramdisk_id=None, **kwargs):
196 super(EBSImageCreator, self).__init__(easy_ec2, instance_id,
197 key_location, description,
198 kernel_id, ramdisk_id)
199 self.name = name
200 self.description = description
201 self.snapshot_description = snapshot_description or description
202 self._snap = None
203 self._vol = None
204
205 @print_timing
207 try:
208 self.clean_private_data()
209 if self.host.root_device_type == "ebs":
210 return self._create_image_from_ebs(size)
211 return self._create_image_from_instance_store(size)
212 except:
213 log.error("Error occurred while creating image")
214 if self._snap:
215 log.error("Removing generated snapshot '%s'" % self._snap)
216 self._snap.delete()
217 if self._vol:
218 log.error("Removing generated volume '%s'" % self._vol.id)
219 self._vol.detach(force=True)
220 self._vol.delete()
221 raise
222
224 log.info("Creating EBS image...")
225 imgid = self.ec2.create_image(self.host.id, self.name,
226 self.description)
227 log.info("Waiting for AMI %s to become available..." % imgid,
228 extra=dict(__nonewline__=True))
229 img = self.ec2.get_image(imgid)
230 s = Spinner()
231 s.start()
232 while img.state == "pending":
233 time.sleep(15)
234 if img.update() == "failed":
235 raise exception.AWSError(
236 "EBS image creation failed for AMI %s" % imgid)
237 s.stop()
238 return imgid
239
241 host = self.host
242 host_ssh = self.host_ssh
243 log.info("Creating new EBS-backed image from instance-store instance")
244 log.info("Creating new root volume...")
245 vol = self._vol = self.ec2.create_volume(size, host.placement)
246 log.info("Created new volume: %s" % vol.id)
247 while vol.update() != 'available':
248 time.sleep(5)
249 dev = None
250 for i in string.ascii_lowercase[::-1]:
251 dev = '/dev/sd%s' % i
252 if not dev in host.block_device_mapping:
253 break
254 log.info("Attaching volume %s to instance %s on %s" %
255 (vol.id, host.id, dev))
256 vol.attach(host.id, dev)
257 while vol.update() != 'in-use':
258 time.sleep(5)
259 while not host_ssh.path_exists(dev):
260 time.sleep(5)
261 host_ssh.execute('mkfs.ext3 -F %s' % dev)
262 mount_point = '/ebs'
263 while host_ssh.path_exists(mount_point):
264 mount_point += '1'
265 host_ssh.mkdir(mount_point)
266 log.info("Mounting %s on %s" % (dev, mount_point))
267 host_ssh.execute('mount %s %s' % (dev, mount_point))
268 log.info("Configuring /etc/fstab")
269 host_ssh.remove_lines_from_file('/etc/fstab', '/mnt')
270 fstab = host_ssh.remote_file('/etc/fstab', 'a')
271 fstab.write('/dev/sdb1 /mnt auto defaults,nobootwait 0 0\n')
272 fstab.close()
273 log.info("Syncing root filesystem to new volume (%s)" % vol.id)
274 host_ssh.execute(
275 'rsync -avx --exclude %(mpt)s --exclude /root/.ssh / %(mpt)s' %
276 {'mpt': mount_point})
277 log.info("Unmounting %s from %s" % (dev, mount_point))
278 host_ssh.execute('umount %s' % mount_point)
279 log.info("Detaching volume %s from %s" % (dev, mount_point))
280 vol.detach()
281 while vol.update() != 'available':
282 time.sleep(5)
283 sdesc = self.snapshot_description
284 snap = self._snap = self.ec2.create_snapshot(vol,
285 description=sdesc,
286 wait_for_snapshot=True)
287 log.info("New snapshot created: %s" % snap.id)
288 log.info("Removing generated volume %s" % vol.id)
289 vol.delete()
290 log.info("Creating root block device map using snapshot %s" % snap.id)
291 bmap = self.ec2.create_root_block_device_map(snap.id,
292 add_ephemeral_drives=True)
293 log.info("Registering new image...")
294 img_id = self.ec2.register_image(name=self.name,
295 description=self.description,
296 architecture=host.architecture,
297 kernel_id=self.kernel_id,
298 ramdisk_id=self.ramdisk_id,
299 root_device_name='/dev/sda1',
300 block_device_map=bmap)
301 return img_id
302
303
304
305 EC2ImageCreator = S3ImageCreator
306