Package starcluster :: Module utils
[hide private]
[frames] | no frames]

Source Code for Module starcluster.utils

  1  #!/usr/bin/env python 
  2  """ 
  3  Utils module for StarCluster 
  4  """ 
  5   
  6  import os 
  7  import re 
  8  import time 
  9  import types 
 10  import inspect 
 11  import calendar 
 12  import urlparse 
 13  import decorator 
 14  from datetime import datetime 
 15   
 16  from starcluster import iptools 
 17  from starcluster import exception 
 18  from starcluster.logger import log 
 19   
 20  try: 
 21      import IPython 
 22      if IPython.__version__ < '0.11': 
 23          from IPython.Shell import IPShellEmbed 
 24          ipy_shell = IPShellEmbed(argv=[]) 
 25      else: 
 26          from IPython import embed 
 27          ipy_shell = lambda local_ns=None: embed(user_ns=local_ns) 
 28  except ImportError: 
 29   
30 - def ipy_shell(local_ns=None):
31 log.error("Unable to load IPython.") 32 log.error("Please check that IPython is installed and working.") 33 log.error("If not, you can install it via: easy_install ipython")
34 35 try: 36 import pudb 37 set_trace = pudb.set_trace 38 except ImportError: 39
40 - def set_trace():
41 log.error("Unable to load PuDB") 42 log.error("Please check that PuDB is installed and working.") 43 log.error("If not, you can install it via: easy_install pudb")
44 45
46 -class AttributeDict(dict):
47 """ 48 Subclass of dict that allows read-only attribute-like access to 49 dictionary key/values 50 """
51 - def __getattr__(self, name):
52 try: 53 return self.__getitem__(name) 54 except KeyError: 55 return super(AttributeDict, self).__getattribute__(name)
56 57 90 91 if type(msg) == types.FunctionType: 92 return decorator.decorator(wrap_f, msg) 93 else: 94 return decorator.decorator(wrap_f) 95 96
97 -def is_valid_device(dev):
98 """ 99 Checks that dev matches the following regular expression: 100 /dev/sd[a-z]$ 101 """ 102 regex = re.compile('/dev/sd[a-z]$') 103 try: 104 return regex.match(dev) is not None 105 except TypeError: 106 return False
107 108
109 -def is_valid_partition(part):
110 """ 111 Checks that part matches the following regular expression: 112 /dev/sd[a-z][1-9][0-9]?$ 113 """ 114 regex = re.compile('/dev/sd[a-z][1-9][0-9]?$') 115 try: 116 return regex.match(part) is not None 117 except TypeError: 118 return False
119 120
121 -def is_valid_bucket_name(bucket_name):
122 """ 123 Check if bucket_name is a valid S3 bucket name (as defined by the AWS 124 docs): 125 126 1. 3 <= len(bucket_name) <= 255 127 2. all chars one of: a-z 0-9 . _ - 128 3. first char one of: a-z 0-9 129 4. name must not be a valid ip 130 """ 131 regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$') 132 if not regex.match(bucket_name): 133 return False 134 if iptools.validate_ip(bucket_name): 135 return False 136 return True
137 138
139 -def is_valid_image_name(image_name):
140 """ 141 Check if image_name is a valid AWS image name (as defined by the AWS docs) 142 143 1. 3<= len(image_name) <=128 144 2. all chars one of: a-z A-Z 0-9 ( ) . - / _ 145 """ 146 regex = re.compile('[\w\(\)\.\-\/_]{3,128}$') 147 try: 148 return regex.match(image_name) is not None 149 except TypeError: 150 return False
151 152
153 -def make_one_liner(script):
154 """ 155 Returns command to execute python script as a one-line python program 156 157 e.g. 158 159 import os 160 script = ''' 161 import os 162 print os.path.exists('hi') 163 ''' 164 os.system(make_one_liner(script)) 165 166 Will print out: 167 168 <module 'os' from ...> 169 False 170 """ 171 return 'python -c "%s"' % script.strip().replace('\n', ';')
172 173
174 -def is_url(url):
175 """ 176 Returns True if the provided string is a valid url 177 """ 178 try: 179 parts = urlparse.urlparse(url) 180 scheme = parts[0] 181 netloc = parts[1] 182 if scheme and netloc: 183 return True 184 else: 185 return False 186 except: 187 return False
188 189
190 -def is_iso_time(iso):
191 """ 192 Returns True if provided time can be parsed in iso format 193 to a datetime tuple 194 """ 195 try: 196 iso_to_datetime_tuple(iso) 197 return True 198 except ValueError: 199 return False
200 201
202 -def iso_to_datetime_tuple(iso):
203 """ 204 Converts an iso time string to a datetime tuple 205 """ 206 #remove timezone 207 iso = iso.split('.')[0] 208 try: 209 return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S") 210 except AttributeError: 211 # python2.4 datetime module doesnt have strptime 212 return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
213 214
215 -def datetime_tuple_to_iso(tup):
216 """ 217 Converts a datetime tuple to iso time string 218 """ 219 iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S") 220 return iso
221 222
223 -def get_elapsed_time(past_time):
224 ptime = iso_to_localtime_tuple(past_time) 225 now = datetime.now() 226 delta = now - ptime 227 timestr = time.strftime("%H:%M:%S", time.gmtime(delta.seconds)) 228 if delta.days != -1: 229 timestr = "%d days, %s" % (delta.days, timestr) 230 return timestr
231 232
233 -def iso_to_unix_time(iso):
234 dtup = iso_to_datetime_tuple(iso) 235 secs = calendar.timegm(dtup.timetuple()) 236 return secs
237 238
239 -def iso_to_javascript_timestamp(iso):
240 """ 241 Convert dates to Javascript timestamps (number of milliseconds since 242 January 1st 1970 UTC) 243 """ 244 secs = iso_to_unix_time(iso) 245 return secs * 1000
246 247
248 -def iso_to_localtime_tuple(iso):
249 secs = iso_to_unix_time(iso) 250 t = time.mktime(time.localtime(secs)) 251 return datetime.fromtimestamp(t)
252 253
254 -def permute(a):
255 """ 256 Returns generator of all permutations of a 257 258 The following code is an in-place permutation of a given list, implemented 259 as a generator. Since it only returns references to the list, the list 260 should not be modified outside the generator. The solution is 261 non-recursive, so uses low memory. Work well also with multiple copies of 262 elements in the input list. 263 264 Retrieved from: 265 http://stackoverflow.com/questions/104420/ \ 266 how-to-generate-all-permutations-of-a-list-in-python 267 """ 268 a.sort() 269 yield list(a) 270 if len(a) <= 1: 271 return 272 first = 0 273 last = len(a) 274 while 1: 275 i = last - 1 276 while 1: 277 i = i - 1 278 if a[i] < a[i + 1]: 279 j = last - 1 280 while not (a[i] < a[j]): 281 j = j - 1 282 # swap the values 283 a[i], a[j] = a[j], a[i] 284 r = a[i + 1:last] 285 r.reverse() 286 a[i + 1:last] = r 287 yield list(a) 288 break 289 if i == first: 290 a.reverse() 291 return
292 293
294 -def has_required(programs):
295 """ 296 Same as check_required but returns False if not all commands exist 297 """ 298 try: 299 return check_required(programs) 300 except exception.CommandNotFound: 301 return False
302 303
304 -def check_required(programs):
305 """ 306 Checks that all commands in the programs list exist. Returns 307 True if all commands exist and raises exception.CommandNotFound if not. 308 """ 309 for prog in programs: 310 if not which(prog): 311 raise exception.CommandNotFound(prog) 312 return True
313 314
315 -def which(program):
316 """ 317 Returns the path to the program provided it exists and 318 is on the system's PATH 319 320 retrieved from code snippet by Jay: 321 322 http://stackoverflow.com/questions/377017/ \ 323 test-if-executable-exists-in-python 324 """ 325 def is_exe(fpath): 326 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
327 fpath, fname = os.path.split(program) 328 if fpath: 329 if is_exe(program): 330 return program 331 else: 332 for path in os.environ["PATH"].split(os.pathsep): 333 exe_file = os.path.join(path, program) 334 if is_exe(exe_file): 335 return exe_file 336 337
338 -def tailf(filename):
339 """ 340 Constantly displays the last lines in filename 341 Similar to 'tail -f' unix command 342 """ 343 #Set the filename and open the file 344 file = open(filename, 'r') 345 346 #Find the size of the file and move to the end 347 st_results = os.stat(filename) 348 st_size = st_results[6] 349 file.seek(st_size) 350 351 while True: 352 where = file.tell() 353 line = file.readline() 354 if not line: 355 time.sleep(1) 356 file.seek(where) 357 continue 358 print line, # already has newline
359 360
361 -def v2fhelper(v, suff, version, weight):
362 parts = v.split(suff) 363 if 2 != len(parts): 364 return v 365 version[4] = weight 366 version[5] = parts[1] 367 return parts[0]
368 369
370 -def version_to_float(v):
371 # This code was written by Krzysztof Kowalczyk (http://blog.kowalczyk.info) 372 # and is placed in public domain. 373 """ 374 Convert a Mozilla-style version string into a floating-point number 375 1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc 376 """ 377 version = [ 378 0, 0, 0, 0, # 4-part numerical revision 379 4, # Alpha, beta, RC or (default) final 380 0, # Alpha, beta, or RC version revision 381 1 # Pre or (default) final 382 ] 383 parts = v.split("pre") 384 if 2 == len(parts): 385 version[6] = 0 386 v = parts[0] 387 388 v = v2fhelper(v, "a", version, 1) 389 v = v2fhelper(v, "b", version, 2) 390 v = v2fhelper(v, "rc", version, 3) 391 392 parts = v.split(".")[:4] 393 for (p, i) in zip(parts, range(len(parts))): 394 version[i] = p 395 ver = float(version[0]) 396 ver += float(version[1]) / 100. 397 ver += float(version[2]) / 10000. 398 ver += float(version[3]) / 1000000. 399 ver += float(version[4]) / 100000000. 400 ver += float(version[5]) / 10000000000. 401 ver += float(version[6]) / 1000000000000. 402 return ver
403 404
405 -def program_version_greater(ver1, ver2):
406 """ 407 Return True if ver1 > ver2 using semantics of comparing version 408 numbers 409 """ 410 v1f = version_to_float(ver1) 411 v2f = version_to_float(ver2) 412 return v1f > v2f
413 414
415 -def test_version_to_float():
416 assert program_version_greater("1", "0.9") 417 assert program_version_greater("0.0.0.2", "0.0.0.1") 418 assert program_version_greater("1.0", "0.9") 419 assert program_version_greater("2.0.1", "2.0.0") 420 assert program_version_greater("2.0.1", "2.0") 421 assert program_version_greater("2.0.1", "2") 422 assert program_version_greater("0.9.1", "0.9.0") 423 assert program_version_greater("0.9.2", "0.9.1") 424 assert program_version_greater("0.9.11", "0.9.2") 425 assert program_version_greater("0.9.12", "0.9.11") 426 assert program_version_greater("0.10", "0.9") 427 assert program_version_greater("2.0", "2.0b35") 428 assert program_version_greater("1.10.3", "1.10.3b3") 429 assert program_version_greater("88", "88a12") 430 assert program_version_greater("0.0.33", "0.0.33rc23") 431 assert program_version_greater("0.91.2", "0.91.1") 432 assert program_version_greater("0.9999", "0.91.1") 433 assert program_version_greater("0.9999", "0.92") 434 assert program_version_greater("0.91.10", "0.91.1") 435 assert program_version_greater("0.92", "0.91.11") 436 assert program_version_greater("0.92", "0.92b1") 437 assert program_version_greater("0.9999", "0.92b3") 438 print("All tests passed")
439 440
441 -def get_arg_spec(func):
442 """ 443 Convenience wrapper around inspect.getargspec 444 445 Returns a tuple whose first element is a list containing the names of all 446 required arguments and whose second element is a list containing the names 447 of all keyword (optional) arguments. 448 """ 449 allargs, varargs, keywords, defaults = inspect.getargspec(func) 450 if 'self' in allargs: 451 allargs.remove('self') # ignore self 452 nargs = len(allargs) 453 ndefaults = 0 454 if defaults: 455 ndefaults = len(defaults) 456 nrequired = nargs - ndefaults 457 args = allargs[:nrequired] 458 kwargs = allargs[nrequired:] 459 log.debug('nargs = %s' % nargs) 460 log.debug('ndefaults = %s' % ndefaults) 461 log.debug('nrequired = %s' % nrequired) 462 log.debug('args = %s' % args) 463 log.debug('kwargs = %s' % kwargs) 464 log.debug('defaults = %s' % str(defaults)) 465 return args, kwargs
466 467
468 -def chunk_list(ls, items=8):
469 """ 470 iterate through 'chunks' of a list. final chunk consists of remaining 471 elements if items does not divide len(ls) evenly. 472 473 items - size of 'chunks' 474 """ 475 itms = [] 476 for i, v in enumerate(ls): 477 if i >= items and i % items == 0: 478 yield itms 479 itms = [v] 480 else: 481 itms.append(v) 482 if itms: 483 yield itms
484