Package starcluster :: Module utils
[hide private]
[frames] | no frames]

Source Code for Module starcluster.utils

  1  """ 
  2  Utils module for StarCluster 
  3  """ 
  4   
  5  import os 
  6  import re 
  7  import time 
  8  import types 
  9  import string 
 10  import random 
 11  import inspect 
 12  import calendar 
 13  import urlparse 
 14  import decorator 
 15  from datetime import datetime 
 16   
 17  from starcluster import iptools 
 18  from starcluster import exception 
 19  from starcluster.logger import log 
 20   
 21  try: 
 22      import IPython 
 23      if IPython.__version__ < '0.11': 
 24          from IPython.Shell import IPShellEmbed 
 25          ipy_shell = IPShellEmbed(argv=[]) 
 26      else: 
 27          from IPython import embed 
 28          ipy_shell = lambda local_ns=None: embed(user_ns=local_ns) 
 29  except ImportError, e: 
 30   
31 - def ipy_shell(local_ns=None):
32 log.error("Unable to load IPython:\n\n%s\n" % e) 33 log.error("Please check that IPython is installed and working.") 34 log.error("If not, you can install it via: easy_install ipython")
35 36 try: 37 import pudb 38 set_trace = pudb.set_trace 39 except ImportError: 40
41 - def set_trace():
42 log.error("Unable to load PuDB") 43 log.error("Please check that PuDB is installed and working.") 44 log.error("If not, you can install it via: easy_install pudb")
45 46
47 -class AttributeDict(dict):
48 """ 49 Subclass of dict that allows read-only attribute-like access to 50 dictionary key/values 51 """
52 - def __getattr__(self, name):
53 try: 54 return self.__getitem__(name) 55 except KeyError: 56 return super(AttributeDict, self).__getattribute__(name)
57 58 91 92 if type(msg) == types.FunctionType: 93 return decorator.decorator(wrap_f, msg) 94 else: 95 return decorator.decorator(wrap_f) 96 97
98 -def is_valid_device(dev):
99 """ 100 Checks that dev matches the following regular expression: 101 /dev/sd[a-z]$ 102 """ 103 regex = re.compile('/dev/sd[a-z]$') 104 try: 105 return regex.match(dev) is not None 106 except TypeError: 107 return False
108 109
110 -def is_valid_partition(part):
111 """ 112 Checks that part matches the following regular expression: 113 /dev/sd[a-z][1-9][0-9]?$ 114 """ 115 regex = re.compile('/dev/sd[a-z][1-9][0-9]?$') 116 try: 117 return regex.match(part) is not None 118 except TypeError: 119 return False
120 121
122 -def is_valid_bucket_name(bucket_name):
123 """ 124 Check if bucket_name is a valid S3 bucket name (as defined by the AWS 125 docs): 126 127 1. 3 <= len(bucket_name) <= 255 128 2. all chars one of: a-z 0-9 . _ - 129 3. first char one of: a-z 0-9 130 4. name must not be a valid ip 131 """ 132 regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$') 133 if not regex.match(bucket_name): 134 return False 135 if iptools.validate_ip(bucket_name): 136 return False 137 return True
138 139
140 -def is_valid_image_name(image_name):
141 """ 142 Check if image_name is a valid AWS image name (as defined by the AWS docs) 143 144 1. 3<= len(image_name) <=128 145 2. all chars one of: a-z A-Z 0-9 ( ) . - / _ 146 """ 147 regex = re.compile('[\w\(\)\.\-\/_]{3,128}$') 148 try: 149 return regex.match(image_name) is not None 150 except TypeError: 151 return False
152 153
154 -def make_one_liner(script):
155 """ 156 Returns command to execute python script as a one-line python program 157 158 e.g. 159 160 import os 161 script = ''' 162 import os 163 print os.path.exists('hi') 164 ''' 165 os.system(make_one_liner(script)) 166 167 Will print out: 168 169 <module 'os' from ...> 170 False 171 """ 172 return 'python -c "%s"' % script.strip().replace('\n', ';')
173 174
175 -def is_url(url):
176 """ 177 Returns True if the provided string is a valid url 178 """ 179 try: 180 parts = urlparse.urlparse(url) 181 scheme = parts[0] 182 netloc = parts[1] 183 if scheme and netloc: 184 return True 185 else: 186 return False 187 except: 188 return False
189 190
191 -def is_iso_time(iso):
192 """ 193 Returns True if provided time can be parsed in iso format 194 to a datetime tuple 195 """ 196 try: 197 iso_to_datetime_tuple(iso) 198 return True 199 except ValueError: 200 return False
201 202
203 -def iso_to_datetime_tuple(iso):
204 """ 205 Converts an iso time string to a datetime tuple 206 """ 207 #remove timezone 208 iso = iso.split('.')[0] 209 try: 210 return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S") 211 except AttributeError: 212 # python2.4 datetime module doesnt have strptime 213 return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
214 215
216 -def datetime_tuple_to_iso(tup):
217 """ 218 Converts a datetime tuple to iso time string 219 """ 220 iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S") 221 return iso
222 223
224 -def get_elapsed_time(past_time):
225 ptime = iso_to_localtime_tuple(past_time) 226 now = datetime.now() 227 delta = now - ptime 228 timestr = time.strftime("%H:%M:%S", time.gmtime(delta.seconds)) 229 if delta.days != -1: 230 timestr = "%d days, %s" % (delta.days, timestr) 231 return timestr
232 233
234 -def iso_to_unix_time(iso):
235 dtup = iso_to_datetime_tuple(iso) 236 secs = calendar.timegm(dtup.timetuple()) 237 return secs
238 239
240 -def iso_to_javascript_timestamp(iso):
241 """ 242 Convert dates to Javascript timestamps (number of milliseconds since 243 January 1st 1970 UTC) 244 """ 245 secs = iso_to_unix_time(iso) 246 return secs * 1000
247 248
249 -def iso_to_localtime_tuple(iso):
250 secs = iso_to_unix_time(iso) 251 t = time.mktime(time.localtime(secs)) 252 return datetime.fromtimestamp(t)
253 254
255 -def permute(a):
256 """ 257 Returns generator of all permutations of a 258 259 The following code is an in-place permutation of a given list, implemented 260 as a generator. Since it only returns references to the list, the list 261 should not be modified outside the generator. The solution is 262 non-recursive, so uses low memory. Work well also with multiple copies of 263 elements in the input list. 264 265 Retrieved from: 266 http://stackoverflow.com/questions/104420/ \ 267 how-to-generate-all-permutations-of-a-list-in-python 268 """ 269 a.sort() 270 yield list(a) 271 if len(a) <= 1: 272 return 273 first = 0 274 last = len(a) 275 while 1: 276 i = last - 1 277 while 1: 278 i = i - 1 279 if a[i] < a[i + 1]: 280 j = last - 1 281 while not (a[i] < a[j]): 282 j = j - 1 283 # swap the values 284 a[i], a[j] = a[j], a[i] 285 r = a[i + 1:last] 286 r.reverse() 287 a[i + 1:last] = r 288 yield list(a) 289 break 290 if i == first: 291 a.reverse() 292 return
293 294
295 -def has_required(programs):
296 """ 297 Same as check_required but returns False if not all commands exist 298 """ 299 try: 300 return check_required(programs) 301 except exception.CommandNotFound: 302 return False
303 304
305 -def check_required(programs):
306 """ 307 Checks that all commands in the programs list exist. Returns 308 True if all commands exist and raises exception.CommandNotFound if not. 309 """ 310 for prog in programs: 311 if not which(prog): 312 raise exception.CommandNotFound(prog) 313 return True
314 315
316 -def which(program):
317 """ 318 Returns the path to the program provided it exists and 319 is on the system's PATH 320 321 retrieved from code snippet by Jay: 322 323 http://stackoverflow.com/questions/377017/ \ 324 test-if-executable-exists-in-python 325 """ 326 def is_exe(fpath): 327 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
328 fpath, fname = os.path.split(program) 329 if fpath: 330 if is_exe(program): 331 return program 332 else: 333 for path in os.environ["PATH"].split(os.pathsep): 334 exe_file = os.path.join(path, program) 335 if is_exe(exe_file): 336 return exe_file 337 338
339 -def tailf(filename):
340 """ 341 Constantly displays the last lines in filename 342 Similar to 'tail -f' unix command 343 """ 344 #Set the filename and open the file 345 file = open(filename, 'r') 346 347 #Find the size of the file and move to the end 348 st_results = os.stat(filename) 349 st_size = st_results[6] 350 file.seek(st_size) 351 352 while True: 353 where = file.tell() 354 line = file.readline() 355 if not line: 356 time.sleep(1) 357 file.seek(where) 358 continue 359 print line, # already has newline
360 361
362 -def v2fhelper(v, suff, version, weight):
363 parts = v.split(suff) 364 if 2 != len(parts): 365 return v 366 version[4] = weight 367 version[5] = parts[1] 368 return parts[0]
369 370
371 -def version_to_float(v):
372 # This code was written by Krzysztof Kowalczyk (http://blog.kowalczyk.info) 373 # and is placed in public domain. 374 """ 375 Convert a Mozilla-style version string into a floating-point number 376 1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc. 377 """ 378 version = [ 379 0, 0, 0, 0, # 4-part numerical revision 380 4, # Alpha, beta, RC or (default) final 381 0, # Alpha, beta, or RC version revision 382 1 # Pre or (default) final 383 ] 384 parts = v.split("pre") 385 if 2 == len(parts): 386 version[6] = 0 387 v = parts[0] 388 389 v = v2fhelper(v, "a", version, 1) 390 v = v2fhelper(v, "b", version, 2) 391 v = v2fhelper(v, "rc", version, 3) 392 393 parts = v.split(".")[:4] 394 for (p, i) in zip(parts, range(len(parts))): 395 version[i] = p 396 ver = float(version[0]) 397 ver += float(version[1]) / 100. 398 ver += float(version[2]) / 10000. 399 ver += float(version[3]) / 1000000. 400 ver += float(version[4]) / 100000000. 401 ver += float(version[5]) / 10000000000. 402 ver += float(version[6]) / 1000000000000. 403 return ver
404 405
406 -def program_version_greater(ver1, ver2):
407 """ 408 Return True if ver1 > ver2 using semantics of comparing version 409 numbers 410 """ 411 v1f = version_to_float(ver1) 412 v2f = version_to_float(ver2) 413 return v1f > v2f
414 415
416 -def test_version_to_float():
417 assert program_version_greater("1", "0.9") 418 assert program_version_greater("0.0.0.2", "0.0.0.1") 419 assert program_version_greater("1.0", "0.9") 420 assert program_version_greater("2.0.1", "2.0.0") 421 assert program_version_greater("2.0.1", "2.0") 422 assert program_version_greater("2.0.1", "2") 423 assert program_version_greater("0.9.1", "0.9.0") 424 assert program_version_greater("0.9.2", "0.9.1") 425 assert program_version_greater("0.9.11", "0.9.2") 426 assert program_version_greater("0.9.12", "0.9.11") 427 assert program_version_greater("0.10", "0.9") 428 assert program_version_greater("2.0", "2.0b35") 429 assert program_version_greater("1.10.3", "1.10.3b3") 430 assert program_version_greater("88", "88a12") 431 assert program_version_greater("0.0.33", "0.0.33rc23") 432 assert program_version_greater("0.91.2", "0.91.1") 433 assert program_version_greater("0.9999", "0.91.1") 434 assert program_version_greater("0.9999", "0.92") 435 assert program_version_greater("0.91.10", "0.91.1") 436 assert program_version_greater("0.92", "0.91.11") 437 assert program_version_greater("0.92", "0.92b1") 438 assert program_version_greater("0.9999", "0.92b3") 439 print("All tests passed")
440 441
442 -def get_arg_spec(func):
443 """ 444 Convenience wrapper around inspect.getargspec 445 446 Returns a tuple whose first element is a list containing the names of all 447 required arguments and whose second element is a list containing the names 448 of all keyword (optional) arguments. 449 """ 450 allargs, varargs, keywords, defaults = inspect.getargspec(func) 451 if 'self' in allargs: 452 allargs.remove('self') # ignore self 453 nargs = len(allargs) 454 ndefaults = 0 455 if defaults: 456 ndefaults = len(defaults) 457 nrequired = nargs - ndefaults 458 args = allargs[:nrequired] 459 kwargs = allargs[nrequired:] 460 log.debug('nargs = %s' % nargs) 461 log.debug('ndefaults = %s' % ndefaults) 462 log.debug('nrequired = %s' % nrequired) 463 log.debug('args = %s' % args) 464 log.debug('kwargs = %s' % kwargs) 465 log.debug('defaults = %s' % str(defaults)) 466 return args, kwargs
467 468
469 -def chunk_list(ls, items=8):
470 """ 471 iterate through 'chunks' of a list. final chunk consists of remaining 472 elements if items does not divide len(ls) evenly. 473 474 items - size of 'chunks' 475 """ 476 itms = [] 477 for i, v in enumerate(ls): 478 if i >= items and i % items == 0: 479 yield itms 480 itms = [v] 481 else: 482 itms.append(v) 483 if itms: 484 yield itms
485 486
487 -def generate_passwd(length):
488 return "".join(random.sample(string.letters + string.digits, length))
489 490
491 -class struct_group(tuple):
492 """ 493 grp.struct_group: Results from getgr*() routines. 494 495 This object may be accessed either as a tuple of 496 (gr_name,gr_passwd,gr_gid,gr_mem) 497 or via the object attributes as named in the above tuple. 498 """ 499 500 attrs = ['gr_name', 'gr_passwd', 'gr_gid', 'gr_mem'] 501
502 - def __new__(cls, grp):
503 if type(grp) not in (list, str, tuple): 504 grp = (grp.name, grp.password, int(grp.GID), 505 [member for member in grp.members]) 506 if len(grp) != 4: 507 raise TypeError('expecting a 4-sequence (%d-sequeunce given)' % 508 len(grp)) 509 return tuple.__new__(cls, grp)
510
511 - def __getattr__(self, attr):
512 try: 513 return self[self.attrs.index(attr)] 514 except ValueError: 515 raise AttributeError
516 517
518 -class struct_passwd(tuple):
519 """ 520 pwd.struct_passwd: Results from getpw*() routines. 521 522 This object may be accessed either as a tuple of 523 (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell) 524 or via the object attributes as named in the above tuple. 525 """ 526 527 attrs = ['pw_name', 'pw_passwd', 'pw_uid', 'pw_gid', 'pw_gecos', 528 'pw_dir', 'pw_shell'] 529
530 - def __new__(cls, pwd):
531 if type(pwd) not in (list, str, tuple): 532 pwd = (pwd.loginName, pwd.password, int(pwd.UID), int(pwd.GID), 533 pwd.GECOS, pwd.home, pwd.shell) 534 if len(pwd) != 7: 535 raise TypeError('expecting a 4-sequence (%d-sequeunce given)' % 536 len(pwd)) 537 return tuple.__new__(cls, pwd)
538
539 - def __getattr__(self, attr):
540 try: 541 return self[self.attrs.index(attr)] 542 except ValueError: 543 raise AttributeError
544