Package starcluster :: Module volume
[hide private]
[frames] | no frames]

Source Code for Module starcluster.volume

  1  #!/usr/bin/env python 
  2  import time 
  3  import string 
  4   
  5  from starcluster import static 
  6  from starcluster import utils 
  7  from starcluster import exception 
  8  from starcluster import cluster 
  9  from starcluster.utils import print_timing 
 10  from starcluster.logger import log 
11 12 13 -class VolumeCreator(cluster.Cluster):
14 """ 15 Handles creating, partitioning, and formatting a new EBS volume. 16 By default this class will format the entire drive (without partitioning) 17 using the ext3 filesystem. 18 19 host_instance - EC2 instance to use when formatting volume. must exist in 20 the same zone as the new volume. if not specified this class will look for 21 host instances in the @sc-volumecreator security group. If it can't find 22 an instance in the @sc-volumecreator group that matches the zone of the 23 new volume, a new instance is launched. 24 25 shutdown_instance - True will shutdown the host instance after volume 26 creation 27 """
28 - def __init__(self, ec2_conn, spot_bid=None, keypair=None, 29 key_location=None, host_instance=None, device='/dev/sdz', 30 image_id=static.BASE_AMI_32, instance_type="m1.small", 31 shutdown_instance=False, detach_vol=False, 32 mkfs_cmd='mkfs.ext3', resizefs_cmd='resize2fs', **kwargs):
33 self._host_instance = host_instance 34 self._instance = None 35 self._volume = None 36 self._device = device or '/dev/sdz' 37 self._image_id = image_id or static.BASE_AMI_32 38 self._instance_type = instance_type or 'm1.small' 39 self._shutdown = shutdown_instance 40 self._detach_vol = detach_vol 41 self._mkfs_cmd = mkfs_cmd 42 self._resizefs_cmd = resizefs_cmd 43 self._alias_tmpl = "volhost-%s" 44 super(VolumeCreator, self).__init__( 45 ec2_conn=ec2_conn, spot_bid=spot_bid, keyname=keypair, 46 key_location=key_location, cluster_tag=static.VOLUME_GROUP_NAME, 47 cluster_size=1, cluster_user="sgeadmin", cluster_shell="bash", 48 node_image_id=self._image_id, 49 node_instance_type=self._instance_type)
50
51 - def __repr__(self):
52 return "<VolumeCreator: %s>" % self._mkfs_cmd
53
54 - def _get_existing_instance(self, zone):
55 """ 56 Returns any existing instance in the @sc-volumecreator group that's 57 located in zone. 58 """ 59 active_states = ['pending', 'running'] 60 i = self._host_instance 61 if i and self._validate_host_instance(i, zone): 62 log.info("Using specified host instance %s" % i.id) 63 return i 64 for node in self.nodes: 65 if node.state in active_states and node.placement == zone: 66 log.info("Using existing instance %s in group %s" % \ 67 (node.id, self.cluster_group.name)) 68 return node
69
70 - def _request_instance(self, zone):
71 self._instance = self._get_existing_instance(zone) 72 if not self._instance: 73 alias = self._alias_tmpl % zone 74 self._validate_image_and_type(self._image_id, self._instance_type) 75 log.info( 76 "No instance in group %s for zone %s, launching one now." % \ 77 (self.cluster_group.name, zone)) 78 self._resv = self.create_node(alias, image_id=self._image_id, 79 instance_type=self._instance_type, 80 zone=zone) 81 self.wait_for_cluster(msg="Waiting for volume host to come up...") 82 self._instance = self.get_node_by_alias(alias) 83 else: 84 s = self.get_spinner("Waiting for instance %s to come up..." % \ 85 self._instance.id) 86 while not self._instance.is_up(): 87 time.sleep(self.refresh_interval) 88 s.stop() 89 return self._instance
90
91 - def _create_volume(self, size, zone, snapshot_id=None):
92 msg = "Creating %sGB volume in zone %s" % (size, zone) 93 if snapshot_id: 94 msg += " from snapshot %s" % snapshot_id 95 log.info(msg) 96 vol = self.ec2.create_volume(size, zone, snapshot_id) 97 log.info("New volume id: %s" % vol.id) 98 s = self.get_spinner("Waiting for new volume to become 'available'...") 99 while vol.status != 'available': 100 time.sleep(5) 101 vol.update() 102 s.stop() 103 self._volume = vol 104 return self._volume
105
106 - def _determine_device(self):
107 block_dev_map = self._instance.block_device_mapping 108 for char in string.lowercase[::-1]: 109 dev = '/dev/sd%s' % char 110 if not block_dev_map.get(dev): 111 self._device = dev 112 return self._device
113
114 - def _attach_volume(self, vol, instance_id, device):
115 s = self.get_spinner("Attaching volume %s to instance %s..." % \ 116 (vol.id, instance_id)) 117 vol.attach(instance_id, device) 118 while True: 119 vol.update() 120 if vol.attachment_state() == 'attached': 121 break 122 time.sleep(5) 123 s.stop() 124 return self._volume
125
126 - def _validate_host_instance(self, instance, zone):
127 if instance.state not in ['pending', 'running']: 128 raise exception.InstanceNotRunning(instance.id) 129 if instance.placement != zone: 130 raise exception.ValidationError( 131 "specified host instance %s is not in zone %s" % 132 (instance.id, zone)) 133 return True
134
135 - def _validate_image_and_type(self, image, itype):
136 img = self.ec2.get_image_or_none(image) 137 if not img: 138 raise exception.ValidationError( 139 'image %s does not exist' % image) 140 if not itype in static.INSTANCE_TYPES: 141 choices = ', '.join(static.INSTANCE_TYPES) 142 raise exception.ValidationError( 143 'instance_type must be one of: %s' % choices) 144 itype_platform = static.INSTANCE_TYPES.get(itype) 145 img_platform = img.architecture 146 if img_platform not in itype_platform: 147 error_msg = "instance_type %(itype)s is for an " + \ 148 "%(iplat)s platform while image_id " + \ 149 "%(img)s is an %(imgplat)s platform" 150 error_dict = {'itype': itype, 'iplat': ', '.join(itype_platform), 151 'img': img.id, 'imgplat': img_platform} 152 raise exception.ValidationError(error_msg % error_dict)
153
154 - def _validate_zone(self, zone):
155 z = self.ec2.get_zone(zone) 156 if z.state != 'available': 157 log.warn('zone %s is not available at this time' % zone) 158 return True
159
160 - def _validate_size(self, size):
161 try: 162 volume_size = int(size) 163 if volume_size < 1: 164 raise exception.ValidationError( 165 "volume_size must be an integer >= 1") 166 except ValueError: 167 raise exception.ValidationError("volume_size must be an integer")
168
169 - def _validate_device(self, device):
170 if not utils.is_valid_device(device): 171 raise exception.ValidationError("volume device %s is not valid" % \ 172 device)
173
174 - def _validate_required_progs(self, progs):
175 log.info("Checking for required remote commands...") 176 self._instance.ssh.check_required(progs)
177
178 - def validate(self, size, zone, device):
179 self._validate_size(size) 180 self._validate_zone(zone) 181 self._validate_device(device)
182
183 - def is_valid(self, size, zone, device):
184 try: 185 self.validate(size, zone, device) 186 return True 187 except exception.BaseException, e: 188 log.error(e.msg) 189 return False
190
191 - def _partition_volume(self):
192 self._instance.ssh.execute('echo ",,L" | sfdisk %s' % self._device, 193 silent=False)
194
195 - def _format_volume(self):
196 log.info("Formatting volume...") 197 self._instance.ssh.execute('%s -F %s' % (self._mkfs_cmd, self._device), 198 silent=False)
199
200 - def _warn_about_volume_hosts(self):
201 sg = self.ec2.get_group_or_none(static.VOLUME_GROUP) 202 if not sg: 203 return 204 vol_hosts = filter(lambda x: x.state in ['running', 'pending'], 205 sg.instances()) 206 vol_hosts = map(lambda x: x.id, vol_hosts) 207 if vol_hosts: 208 log.warn("There are still volume hosts running: %s" % \ 209 ', '.join(vol_hosts)) 210 log.warn(("Run 'starcluster terminate %s' to terminate *all* " + \ 211 "volume host instances once they're no longer needed") % \ 212 static.VOLUME_GROUP_NAME) 213 else: 214 log.info("No active volume hosts found. Run 'starcluster " + \ 215 "terminate %(g)s' to remove the '%(g)s' group" % \ 216 {'g': static.VOLUME_GROUP_NAME})
217
218 - def shutdown(self):
219 vol = self._volume 220 host = self._instance 221 if self._detach_vol: 222 log.info("Detaching volume %s from instance %s" % \ 223 (vol.id, host.id)) 224 vol.detach() 225 else: 226 log.info("Leaving volume %s attached to instance %s" % \ 227 (vol.id, host.id)) 228 if self._shutdown: 229 log.info("Terminating host instance %s" % host.id) 230 host.terminate() 231 else: 232 log.info("Not terminating host instance %s" % \ 233 host.id)
234 235 @print_timing("Creating volume")
236 - def create(self, volume_size, volume_zone):
237 try: 238 self.validate(volume_size, volume_zone, self._device) 239 instance = self._request_instance(volume_zone) 240 self._validate_required_progs([self._mkfs_cmd.split()[0]]) 241 self._determine_device() 242 vol = self._create_volume(volume_size, volume_zone) 243 self._attach_volume(self._volume, instance.id, self._device) 244 self._format_volume() 245 self.shutdown() 246 self._warn_about_volume_hosts() 247 return vol.id 248 except Exception: 249 if self._volume: 250 log.error( 251 "Error occured. Detaching, and deleting volume: %s" % \ 252 self._volume.id) 253 self._volume.detach(force=True) 254 time.sleep(5) 255 self._volume.delete() 256 self._warn_about_volume_hosts() 257 raise
258
259 - def _validate_resize(self, vol, size):
260 self._validate_size(size) 261 if vol.size > size: 262 log.warn("You are attempting to shrink an EBS volume. " + \ 263 "Data loss may occur")
264
265 - def resize(self, vol, size, dest_zone=None):
266 """ 267 Resize EBS volume 268 269 vol - boto volume object 270 size - new volume sze 271 dest_zone - zone to create the new resized volume in. this must be 272 within the original volume's region otherwise a manual copy (rsync) 273 is required. this is currently not implemented. 274 """ 275 try: 276 self._validate_device(self._device) 277 self._validate_resize(vol, size) 278 zone = vol.zone 279 if dest_zone: 280 self._validate_zone(dest_zone) 281 zone = dest_zone 282 host = self._request_instance(zone) 283 self._validate_required_progs([self._resizefs_cmd.split()[0]]) 284 self._determine_device() 285 snap = self.ec2.create_snapshot(vol, wait_for_snapshot=True) 286 new_vol = self._create_volume(size, zone, snap.id) 287 self._attach_volume(new_vol, host.id, self._device) 288 devs = filter(lambda x: x.startswith(self._device), 289 host.ssh.ls('/dev')) 290 device = self._device 291 if len(devs) == 1: 292 log.info("No partitions found, resizing entire device") 293 elif len(devs) == 2: 294 log.info("One partition found, resizing partition...") 295 self._partition_volume() 296 device += '1' 297 else: 298 raise exception.InvalidOperation( 299 "EBS volume %s has more than 1 partition. " 300 "You must resize this volume manually" % vol.id) 301 host.ssh.execute(' '.join([self._resizefs_cmd, device])) 302 log.info("Removing generated snapshot %s" % snap.id) 303 snap.delete() 304 self.shutdown() 305 self._warn_about_volume_hosts() 306 return new_vol.id 307 except Exception: 308 self._warn_about_volume_hosts() 309 raise
310