Package starcluster :: Module utils
[hide private]
[frames] | no frames]

Source Code for Module starcluster.utils

  1  #!/usr/bin/env python 
  2  """ 
  3  Utils module for StarCluster 
  4  """ 
  5   
  6  import os 
  7  import re 
  8  import time 
  9  import types 
 10  import calendar 
 11  import urlparse 
 12  from datetime import datetime 
 13   
 14  from starcluster import iptools 
 15  from starcluster import exception 
 16  from starcluster.logger import log 
 17   
 18   
19 -class AttributeDict(dict):
20 """ 21 Subclass of dict that allows read-only attribute-like access to 22 dictionary key/values 23 """
24 - def __getattr__(self, name):
25 try: 26 return self.__getitem__(name) 27 except KeyError: 28 return super(AttributeDict, self).__getattribute__(name)
29 30 63 return wrap_f 64 65 def wrap(func): 66 def wrap_f(*arg, **kargs): 67 """Raw timing function """ 68 time1 = time.time() 69 res = func(*arg, **kargs) 70 time2 = time.time() 71 prefix = msg 72 log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0)) 73 return res 74 return wrap_f 75 return wrap 76 77
78 -def is_valid_device(dev):
79 """ 80 Checks that dev matches the following regular expression: 81 /dev/sd[a-z]$ 82 """ 83 regex = re.compile('/dev/sd[a-z]$') 84 try: 85 return regex.match(dev) is not None 86 except TypeError: 87 return False
88 89
90 -def is_valid_partition(part):
91 """ 92 Checks that part matches the following regular expression: 93 /dev/sd[a-z][1-9][0-9]?$ 94 """ 95 regex = re.compile('/dev/sd[a-z][1-9][0-9]?$') 96 try: 97 return regex.match(part) is not None 98 except TypeError: 99 return False
100 101
102 -def is_valid_bucket_name(bucket_name):
103 """ 104 Check if bucket_name is a valid S3 bucket name (as defined by the AWS 105 docs): 106 107 1. 3 <= len(bucket_name) <= 255 108 2. all chars one of: a-z 0-9 . _ - 109 3. first char one of: a-z 0-9 110 4. name must not be a valid ip 111 """ 112 regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$') 113 if not regex.match(bucket_name): 114 return False 115 if iptools.validate_ip(bucket_name): 116 return False 117 return True
118 119
120 -def is_valid_image_name(image_name):
121 """ 122 Check if image_name is a valid AWS image name (as defined by the AWS docs) 123 124 1. 3<= len(image_name) <=128 125 2. all chars one of: a-z A-Z 0-9 ( ) . - / _ 126 """ 127 regex = re.compile('[\w\(\)\.\-\/_]{3,128}$') 128 try: 129 return regex.match(image_name) is not None 130 except TypeError: 131 return False
132 133
134 -def make_one_liner(script):
135 """ 136 Returns command to execute python script as a one-line python program 137 138 e.g. 139 140 import os 141 script = ''' 142 import os 143 print os.path.exists('hi') 144 ''' 145 os.system(make_one_liner(script)) 146 147 Will print out: 148 149 <module 'os' from ...> 150 False 151 """ 152 return 'python -c "%s"' % script.strip().replace('\n', ';')
153 154
155 -def is_url(url):
156 """ 157 Returns True if the provided string is a valid url 158 """ 159 try: 160 parts = urlparse.urlparse(url) 161 scheme = parts[0] 162 netloc = parts[1] 163 if scheme and netloc: 164 return True 165 else: 166 return False 167 except: 168 return False
169 170
171 -def is_iso_time(iso):
172 """ 173 Returns True if provided time can be parsed in iso format 174 to a datetime tuple 175 """ 176 try: 177 iso_to_datetime_tuple(iso) 178 return True 179 except ValueError: 180 return False
181 182
183 -def iso_to_datetime_tuple(iso):
184 """ 185 Converts an iso time string to a datetime tuple 186 """ 187 #remove timezone 188 iso = iso.split('.')[0] 189 try: 190 return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S") 191 except AttributeError: 192 # python2.4 datetime module doesnt have strptime 193 return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
194 195
196 -def datetime_tuple_to_iso(tup):
197 """ 198 Converts a datetime tuple to iso time string 199 """ 200 iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S") 201 return iso
202 203
204 -def get_elapsed_time(past_time):
205 ptime = iso_to_localtime_tuple(past_time) 206 now = datetime.now() 207 delta = now - ptime 208 return time.strftime("%H:%M:%S", time.gmtime(delta.seconds))
209 210
211 -def iso_to_localtime_tuple(iso):
212 dtup = iso_to_datetime_tuple(iso) 213 secs = calendar.timegm(dtup.timetuple()) 214 t = time.mktime(time.localtime(secs)) 215 return datetime.fromtimestamp(t)
216 217 try: 218 import IPython.Shell 219 ipy_shell = IPython.Shell.IPShellEmbed(argv=[]) 220 except ImportError: 221
222 - def ipy_shell(local_ns=None):
223 log.error("Unable to load IPython.") 224 log.error("Please check that IPython is installed and working.") 225 log.error("If not, you can install it via: easy_install ipython")
226 227
228 -def permute(a):
229 """ 230 Returns generator of all permutations of a 231 232 The following code is an in-place permutation of a given list, implemented 233 as a generator. Since it only returns references to the list, the list 234 should not be modified outside the generator. The solution is 235 non-recursive, so uses low memory. Work well also with multiple copies of 236 elements in the input list. 237 238 Retrieved from: 239 http://stackoverflow.com/questions/104420/ \ 240 how-to-generate-all-permutations-of-a-list-in-python 241 """ 242 a.sort() 243 yield list(a) 244 if len(a) <= 1: 245 return 246 first = 0 247 last = len(a) 248 while 1: 249 i = last - 1 250 while 1: 251 i = i - 1 252 if a[i] < a[i + 1]: 253 j = last - 1 254 while not (a[i] < a[j]): 255 j = j - 1 256 # swap the values 257 a[i], a[j] = a[j], a[i] 258 r = a[i + 1:last] 259 r.reverse() 260 a[i + 1:last] = r 261 yield list(a) 262 break 263 if i == first: 264 a.reverse() 265 return
266 267
268 -def has_required(programs):
269 """ 270 Same as check_required but returns False if not all commands exist 271 """ 272 try: 273 return check_required(programs) 274 except exception.CommandNotFound: 275 return False
276 277
278 -def check_required(programs):
279 """ 280 Checks that all commands in the programs list exist. Returns 281 True if all commands exist and raises exception.CommandNotFound if not. 282 """ 283 for prog in programs: 284 if not which(prog): 285 raise exception.CommandNotFound(prog) 286 return True
287 288
289 -def which(program):
290 """ 291 Returns the path to the program provided it exists and 292 is on the system's PATH 293 294 retrieved from code snippet by Jay: 295 296 http://stackoverflow.com/questions/377017/ \ 297 test-if-executable-exists-in-python 298 """ 299 def is_exe(fpath): 300 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
301 fpath, fname = os.path.split(program) 302 if fpath: 303 if is_exe(program): 304 return program 305 else: 306 for path in os.environ["PATH"].split(os.pathsep): 307 exe_file = os.path.join(path, program) 308 if is_exe(exe_file): 309 return exe_file 310 311
312 -def tailf(filename):
313 """ 314 Constantly displays the last lines in filename 315 Similar to 'tail -f' unix command 316 """ 317 #Set the filename and open the file 318 file = open(filename, 'r') 319 320 #Find the size of the file and move to the end 321 st_results = os.stat(filename) 322 st_size = st_results[6] 323 file.seek(st_size) 324 325 while True: 326 where = file.tell() 327 line = file.readline() 328 if not line: 329 time.sleep(1) 330 file.seek(where) 331 continue 332 print line, # already has newline
333 334
335 -def v2fhelper(v, suff, version, weight):
336 parts = v.split(suff) 337 if 2 != len(parts): 338 return v 339 version[4] = weight 340 version[5] = parts[1] 341 return parts[0]
342 343
344 -def version_to_float(v):
345 # This code was written by Krzysztof Kowalczyk (http://blog.kowalczyk.info) 346 # and is placed in public domain. 347 """ 348 Convert a Mozilla-style version string into a floating-point number 349 1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc 350 """ 351 version = [ 352 0, 0, 0, 0, # 4-part numerical revision 353 4, # Alpha, beta, RC or (default) final 354 0, # Alpha, beta, or RC version revision 355 1 # Pre or (default) final 356 ] 357 parts = v.split("pre") 358 if 2 == len(parts): 359 version[6] = 0 360 v = parts[0] 361 362 v = v2fhelper(v, "a", version, 1) 363 v = v2fhelper(v, "b", version, 2) 364 v = v2fhelper(v, "rc", version, 3) 365 366 parts = v.split(".")[:4] 367 for (p, i) in zip(parts, range(len(parts))): 368 version[i] = p 369 ver = float(version[0]) 370 ver += float(version[1]) / 100. 371 ver += float(version[2]) / 10000. 372 ver += float(version[3]) / 1000000. 373 ver += float(version[4]) / 100000000. 374 ver += float(version[5]) / 10000000000. 375 ver += float(version[6]) / 1000000000000. 376 return ver
377 378
379 -def program_version_greater(ver1, ver2):
380 """ 381 Return True if ver1 > ver2 using semantics of comparing version 382 numbers 383 """ 384 v1f = version_to_float(ver1) 385 v2f = version_to_float(ver2) 386 return v1f > v2f
387 388
389 -def test_version_to_float():
390 assert program_version_greater("1", "0.9") 391 assert program_version_greater("0.0.0.2", "0.0.0.1") 392 assert program_version_greater("1.0", "0.9") 393 assert program_version_greater("2.0.1", "2.0.0") 394 assert program_version_greater("2.0.1", "2.0") 395 assert program_version_greater("2.0.1", "2") 396 assert program_version_greater("0.9.1", "0.9.0") 397 assert program_version_greater("0.9.2", "0.9.1") 398 assert program_version_greater("0.9.11", "0.9.2") 399 assert program_version_greater("0.9.12", "0.9.11") 400 assert program_version_greater("0.10", "0.9") 401 assert program_version_greater("2.0", "2.0b35") 402 assert program_version_greater("1.10.3", "1.10.3b3") 403 assert program_version_greater("88", "88a12") 404 assert program_version_greater("0.0.33", "0.0.33rc23") 405 assert program_version_greater("0.91.2", "0.91.1") 406 assert program_version_greater("0.9999", "0.91.1") 407 assert program_version_greater("0.9999", "0.92") 408 assert program_version_greater("0.91.10", "0.91.1") 409 assert program_version_greater("0.92", "0.91.11") 410 assert program_version_greater("0.92", "0.92b1") 411 assert program_version_greater("0.9999", "0.92b3") 412 print("All tests passed")
413