1 """
2 Utils module for StarCluster
3 """
4
5 import os
6 import re
7 import time
8 import types
9 import string
10 import random
11 import inspect
12 import calendar
13 import urlparse
14 import decorator
15 from datetime import datetime
16
17 from starcluster import iptools
18 from starcluster import exception
19 from starcluster.logger import log
20
21 try:
22 import IPython
23 if IPython.__version__ < '0.11':
24 from IPython.Shell import IPShellEmbed
25 ipy_shell = IPShellEmbed(argv=[])
26 else:
27 from IPython import embed
28 ipy_shell = lambda local_ns=None: embed(user_ns=local_ns)
29 except ImportError, e:
30
32 log.error("Unable to load IPython:\n\n%s\n" % e)
33 log.error("Please check that IPython is installed and working.")
34 log.error("If not, you can install it via: easy_install ipython")
35
36 try:
37 import pudb
38 set_trace = pudb.set_trace
39 except ImportError:
40
42 log.error("Unable to load PuDB")
43 log.error("Please check that PuDB is installed and working.")
44 log.error("If not, you can install it via: easy_install pudb")
45
46
48 """
49 Subclass of dict that allows read-only attribute-like access to
50 dictionary key/values
51 """
53 try:
54 return self.__getitem__(name)
55 except KeyError:
56 return super(AttributeDict, self).__getattribute__(name)
57
58
60 """
61 Decorator for printing execution time (in mins) of a function
62 Optionally takes a user-friendly msg as argument. This msg will
63 appear in the sentence "[msg] took XXX mins". If no msg is specified,
64 msg will default to the decorated function's name. e.g:
65
66 @print_timing
67 def myfunc():
68 print 'Running myfunc'
69 >>> myfunc()
70 Running myfunc
71 myfunc took 0.000 mins
72
73 @print_timing('My function')
74 def myfunc():
75 print 'Running myfunc'
76 >>> myfunc()
77 Running myfunc
78 My function took 0.000 mins
79 """
80 prefix = msg
81 if type(msg) == types.FunctionType:
82 prefix = msg.func_name
83
84 def wrap_f(func, *arg, **kargs):
85 """Raw timing function """
86 time1 = time.time()
87 res = func(*arg, **kargs)
88 time2 = time.time()
89 log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
90 return res
91
92 if type(msg) == types.FunctionType:
93 return decorator.decorator(wrap_f, msg)
94 else:
95 return decorator.decorator(wrap_f)
96
97
99 """
100 Checks that dev matches the following regular expression:
101 /dev/sd[a-z]$
102 """
103 regex = re.compile('/dev/sd[a-z]$')
104 try:
105 return regex.match(dev) is not None
106 except TypeError:
107 return False
108
109
111 """
112 Checks that part matches the following regular expression:
113 /dev/sd[a-z][1-9][0-9]?$
114 """
115 regex = re.compile('/dev/sd[a-z][1-9][0-9]?$')
116 try:
117 return regex.match(part) is not None
118 except TypeError:
119 return False
120
121
123 """
124 Check if bucket_name is a valid S3 bucket name (as defined by the AWS
125 docs):
126
127 1. 3 <= len(bucket_name) <= 255
128 2. all chars one of: a-z 0-9 . _ -
129 3. first char one of: a-z 0-9
130 4. name must not be a valid ip
131 """
132 regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$')
133 if not regex.match(bucket_name):
134 return False
135 if iptools.validate_ip(bucket_name):
136 return False
137 return True
138
139
141 """
142 Check if image_name is a valid AWS image name (as defined by the AWS docs)
143
144 1. 3<= len(image_name) <=128
145 2. all chars one of: a-z A-Z 0-9 ( ) . - / _
146 """
147 regex = re.compile('[\w\(\)\.\-\/_]{3,128}$')
148 try:
149 return regex.match(image_name) is not None
150 except TypeError:
151 return False
152
153
155 """
156 Returns command to execute python script as a one-line python program
157
158 e.g.
159
160 import os
161 script = '''
162 import os
163 print os.path.exists('hi')
164 '''
165 os.system(make_one_liner(script))
166
167 Will print out:
168
169 <module 'os' from ...>
170 False
171 """
172 return 'python -c "%s"' % script.strip().replace('\n', ';')
173
174
176 """
177 Returns True if the provided string is a valid url
178 """
179 try:
180 parts = urlparse.urlparse(url)
181 scheme = parts[0]
182 netloc = parts[1]
183 if scheme and netloc:
184 return True
185 else:
186 return False
187 except:
188 return False
189
190
192 """
193 Returns True if provided time can be parsed in iso format
194 to a datetime tuple
195 """
196 try:
197 iso_to_datetime_tuple(iso)
198 return True
199 except ValueError:
200 return False
201
202
204 """
205 Converts an iso time string to a datetime tuple
206 """
207
208 iso = iso.split('.')[0]
209 try:
210 return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S")
211 except AttributeError:
212
213 return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
214
215
217 """
218 Converts a datetime tuple to iso time string
219 """
220 iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S")
221 return iso
222
223
225 ptime = iso_to_localtime_tuple(past_time)
226 now = datetime.now()
227 delta = now - ptime
228 timestr = time.strftime("%H:%M:%S", time.gmtime(delta.seconds))
229 if delta.days != -1:
230 timestr = "%d days, %s" % (delta.days, timestr)
231 return timestr
232
233
238
239
241 """
242 Convert dates to Javascript timestamps (number of milliseconds since
243 January 1st 1970 UTC)
244 """
245 secs = iso_to_unix_time(iso)
246 return secs * 1000
247
248
250 secs = iso_to_unix_time(iso)
251 t = time.mktime(time.localtime(secs))
252 return datetime.fromtimestamp(t)
253
254
256 """
257 Returns generator of all permutations of a
258
259 The following code is an in-place permutation of a given list, implemented
260 as a generator. Since it only returns references to the list, the list
261 should not be modified outside the generator. The solution is
262 non-recursive, so uses low memory. Work well also with multiple copies of
263 elements in the input list.
264
265 Retrieved from:
266 http://stackoverflow.com/questions/104420/ \
267 how-to-generate-all-permutations-of-a-list-in-python
268 """
269 a.sort()
270 yield list(a)
271 if len(a) <= 1:
272 return
273 first = 0
274 last = len(a)
275 while 1:
276 i = last - 1
277 while 1:
278 i = i - 1
279 if a[i] < a[i + 1]:
280 j = last - 1
281 while not (a[i] < a[j]):
282 j = j - 1
283
284 a[i], a[j] = a[j], a[i]
285 r = a[i + 1:last]
286 r.reverse()
287 a[i + 1:last] = r
288 yield list(a)
289 break
290 if i == first:
291 a.reverse()
292 return
293
294
303
304
306 """
307 Checks that all commands in the programs list exist. Returns
308 True if all commands exist and raises exception.CommandNotFound if not.
309 """
310 for prog in programs:
311 if not which(prog):
312 raise exception.CommandNotFound(prog)
313 return True
314
315
317 """
318 Returns the path to the program provided it exists and
319 is on the system's PATH
320
321 retrieved from code snippet by Jay:
322
323 http://stackoverflow.com/questions/377017/ \
324 test-if-executable-exists-in-python
325 """
326 def is_exe(fpath):
327 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
328 fpath, fname = os.path.split(program)
329 if fpath:
330 if is_exe(program):
331 return program
332 else:
333 for path in os.environ["PATH"].split(os.pathsep):
334 exe_file = os.path.join(path, program)
335 if is_exe(exe_file):
336 return exe_file
337
338
340 """
341 Constantly displays the last lines in filename
342 Similar to 'tail -f' unix command
343 """
344
345 file = open(filename, 'r')
346
347
348 st_results = os.stat(filename)
349 st_size = st_results[6]
350 file.seek(st_size)
351
352 while True:
353 where = file.tell()
354 line = file.readline()
355 if not line:
356 time.sleep(1)
357 file.seek(where)
358 continue
359 print line,
360
361
363 parts = v.split(suff)
364 if 2 != len(parts):
365 return v
366 version[4] = weight
367 version[5] = parts[1]
368 return parts[0]
369
370
372
373
374 """
375 Convert a Mozilla-style version string into a floating-point number
376 1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc.
377 """
378 version = [
379 0, 0, 0, 0,
380 4,
381 0,
382 1
383 ]
384 parts = v.split("pre")
385 if 2 == len(parts):
386 version[6] = 0
387 v = parts[0]
388
389 v = v2fhelper(v, "a", version, 1)
390 v = v2fhelper(v, "b", version, 2)
391 v = v2fhelper(v, "rc", version, 3)
392
393 parts = v.split(".")[:4]
394 for (p, i) in zip(parts, range(len(parts))):
395 version[i] = p
396 ver = float(version[0])
397 ver += float(version[1]) / 100.
398 ver += float(version[2]) / 10000.
399 ver += float(version[3]) / 1000000.
400 ver += float(version[4]) / 100000000.
401 ver += float(version[5]) / 10000000000.
402 ver += float(version[6]) / 1000000000000.
403 return ver
404
405
407 """
408 Return True if ver1 > ver2 using semantics of comparing version
409 numbers
410 """
411 v1f = version_to_float(ver1)
412 v2f = version_to_float(ver2)
413 return v1f > v2f
414
415
417 assert program_version_greater("1", "0.9")
418 assert program_version_greater("0.0.0.2", "0.0.0.1")
419 assert program_version_greater("1.0", "0.9")
420 assert program_version_greater("2.0.1", "2.0.0")
421 assert program_version_greater("2.0.1", "2.0")
422 assert program_version_greater("2.0.1", "2")
423 assert program_version_greater("0.9.1", "0.9.0")
424 assert program_version_greater("0.9.2", "0.9.1")
425 assert program_version_greater("0.9.11", "0.9.2")
426 assert program_version_greater("0.9.12", "0.9.11")
427 assert program_version_greater("0.10", "0.9")
428 assert program_version_greater("2.0", "2.0b35")
429 assert program_version_greater("1.10.3", "1.10.3b3")
430 assert program_version_greater("88", "88a12")
431 assert program_version_greater("0.0.33", "0.0.33rc23")
432 assert program_version_greater("0.91.2", "0.91.1")
433 assert program_version_greater("0.9999", "0.91.1")
434 assert program_version_greater("0.9999", "0.92")
435 assert program_version_greater("0.91.10", "0.91.1")
436 assert program_version_greater("0.92", "0.91.11")
437 assert program_version_greater("0.92", "0.92b1")
438 assert program_version_greater("0.9999", "0.92b3")
439 print("All tests passed")
440
441
443 """
444 Convenience wrapper around inspect.getargspec
445
446 Returns a tuple whose first element is a list containing the names of all
447 required arguments and whose second element is a list containing the names
448 of all keyword (optional) arguments.
449 """
450 allargs, varargs, keywords, defaults = inspect.getargspec(func)
451 if 'self' in allargs:
452 allargs.remove('self')
453 nargs = len(allargs)
454 ndefaults = 0
455 if defaults:
456 ndefaults = len(defaults)
457 nrequired = nargs - ndefaults
458 args = allargs[:nrequired]
459 kwargs = allargs[nrequired:]
460 log.debug('nargs = %s' % nargs)
461 log.debug('ndefaults = %s' % ndefaults)
462 log.debug('nrequired = %s' % nrequired)
463 log.debug('args = %s' % args)
464 log.debug('kwargs = %s' % kwargs)
465 log.debug('defaults = %s' % str(defaults))
466 return args, kwargs
467
468
470 """
471 iterate through 'chunks' of a list. final chunk consists of remaining
472 elements if items does not divide len(ls) evenly.
473
474 items - size of 'chunks'
475 """
476 itms = []
477 for i, v in enumerate(ls):
478 if i >= items and i % items == 0:
479 yield itms
480 itms = [v]
481 else:
482 itms.append(v)
483 if itms:
484 yield itms
485
486
488 return "".join(random.sample(string.letters + string.digits, length))
489
490
492 """
493 grp.struct_group: Results from getgr*() routines.
494
495 This object may be accessed either as a tuple of
496 (gr_name,gr_passwd,gr_gid,gr_mem)
497 or via the object attributes as named in the above tuple.
498 """
499
500 attrs = ['gr_name', 'gr_passwd', 'gr_gid', 'gr_mem']
501
503 if type(grp) not in (list, str, tuple):
504 grp = (grp.name, grp.password, int(grp.GID),
505 [member for member in grp.members])
506 if len(grp) != 4:
507 raise TypeError('expecting a 4-sequence (%d-sequeunce given)' %
508 len(grp))
509 return tuple.__new__(cls, grp)
510
512 try:
513 return self[self.attrs.index(attr)]
514 except ValueError:
515 raise AttributeError
516
517
519 """
520 pwd.struct_passwd: Results from getpw*() routines.
521
522 This object may be accessed either as a tuple of
523 (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell)
524 or via the object attributes as named in the above tuple.
525 """
526
527 attrs = ['pw_name', 'pw_passwd', 'pw_uid', 'pw_gid', 'pw_gecos',
528 'pw_dir', 'pw_shell']
529
531 if type(pwd) not in (list, str, tuple):
532 pwd = (pwd.loginName, pwd.password, int(pwd.UID), int(pwd.GID),
533 pwd.GECOS, pwd.home, pwd.shell)
534 if len(pwd) != 7:
535 raise TypeError('expecting a 4-sequence (%d-sequeunce given)' %
536 len(pwd))
537 return tuple.__new__(cls, pwd)
538
540 try:
541 return self[self.attrs.index(attr)]
542 except ValueError:
543 raise AttributeError
544