1
2 """
3 Utils module for StarCluster
4 """
5
6 import os
7 import re
8 import time
9 import types
10 import inspect
11 import calendar
12 import urlparse
13 import decorator
14 from datetime import datetime
15
16 from starcluster import iptools
17 from starcluster import exception
18 from starcluster.logger import log
19
20 try:
21 import IPython
22 if IPython.__version__ < '0.11':
23 from IPython.Shell import IPShellEmbed
24 ipy_shell = IPShellEmbed(argv=[])
25 else:
26 from IPython import embed
27 ipy_shell = lambda local_ns=None: embed(user_ns=local_ns)
28 except ImportError:
29
31 log.error("Unable to load IPython.")
32 log.error("Please check that IPython is installed and working.")
33 log.error("If not, you can install it via: easy_install ipython")
34
35 try:
36 import pudb
37 set_trace = pudb.set_trace
38 except ImportError:
39
41 log.error("Unable to load PuDB")
42 log.error("Please check that PuDB is installed and working.")
43 log.error("If not, you can install it via: easy_install pudb")
44
45
47 """
48 Subclass of dict that allows read-only attribute-like access to
49 dictionary key/values
50 """
52 try:
53 return self.__getitem__(name)
54 except KeyError:
55 return super(AttributeDict, self).__getattribute__(name)
56
57
59 """
60 Decorator for printing execution time (in mins) of a function
61 Optionally takes a user-friendly msg as argument. This msg will
62 appear in the sentence "[msg] took XXX mins". If no msg is specified,
63 msg will default to the decorated function's name. e.g:
64
65 @print_timing
66 def myfunc():
67 print 'Running myfunc'
68 >>> myfunc()
69 Running myfunc
70 myfunc took 0.000 mins
71
72 @print_timing('My function')
73 def myfunc():
74 print 'Running myfunc'
75 >>> myfunc()
76 Running myfunc
77 My function took 0.000 mins
78 """
79 prefix = msg
80 if type(msg) == types.FunctionType:
81 prefix = msg.func_name
82
83 def wrap_f(func, *arg, **kargs):
84 """Raw timing function """
85 time1 = time.time()
86 res = func(*arg, **kargs)
87 time2 = time.time()
88 log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
89 return res
90
91 if type(msg) == types.FunctionType:
92 return decorator.decorator(wrap_f, msg)
93 else:
94 return decorator.decorator(wrap_f)
95
96
98 """
99 Checks that dev matches the following regular expression:
100 /dev/sd[a-z]$
101 """
102 regex = re.compile('/dev/sd[a-z]$')
103 try:
104 return regex.match(dev) is not None
105 except TypeError:
106 return False
107
108
110 """
111 Checks that part matches the following regular expression:
112 /dev/sd[a-z][1-9][0-9]?$
113 """
114 regex = re.compile('/dev/sd[a-z][1-9][0-9]?$')
115 try:
116 return regex.match(part) is not None
117 except TypeError:
118 return False
119
120
122 """
123 Check if bucket_name is a valid S3 bucket name (as defined by the AWS
124 docs):
125
126 1. 3 <= len(bucket_name) <= 255
127 2. all chars one of: a-z 0-9 . _ -
128 3. first char one of: a-z 0-9
129 4. name must not be a valid ip
130 """
131 regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$')
132 if not regex.match(bucket_name):
133 return False
134 if iptools.validate_ip(bucket_name):
135 return False
136 return True
137
138
140 """
141 Check if image_name is a valid AWS image name (as defined by the AWS docs)
142
143 1. 3<= len(image_name) <=128
144 2. all chars one of: a-z A-Z 0-9 ( ) . - / _
145 """
146 regex = re.compile('[\w\(\)\.\-\/_]{3,128}$')
147 try:
148 return regex.match(image_name) is not None
149 except TypeError:
150 return False
151
152
154 """
155 Returns command to execute python script as a one-line python program
156
157 e.g.
158
159 import os
160 script = '''
161 import os
162 print os.path.exists('hi')
163 '''
164 os.system(make_one_liner(script))
165
166 Will print out:
167
168 <module 'os' from ...>
169 False
170 """
171 return 'python -c "%s"' % script.strip().replace('\n', ';')
172
173
175 """
176 Returns True if the provided string is a valid url
177 """
178 try:
179 parts = urlparse.urlparse(url)
180 scheme = parts[0]
181 netloc = parts[1]
182 if scheme and netloc:
183 return True
184 else:
185 return False
186 except:
187 return False
188
189
191 """
192 Returns True if provided time can be parsed in iso format
193 to a datetime tuple
194 """
195 try:
196 iso_to_datetime_tuple(iso)
197 return True
198 except ValueError:
199 return False
200
201
203 """
204 Converts an iso time string to a datetime tuple
205 """
206
207 iso = iso.split('.')[0]
208 try:
209 return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S")
210 except AttributeError:
211
212 return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
213
214
216 """
217 Converts a datetime tuple to iso time string
218 """
219 iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S")
220 return iso
221
222
224 ptime = iso_to_localtime_tuple(past_time)
225 now = datetime.now()
226 delta = now - ptime
227 timestr = time.strftime("%H:%M:%S", time.gmtime(delta.seconds))
228 if delta.days != -1:
229 timestr = "%d days, %s" % (delta.days, timestr)
230 return timestr
231
232
237
238
240 """
241 Convert dates to Javascript timestamps (number of milliseconds since
242 January 1st 1970 UTC)
243 """
244 secs = iso_to_unix_time(iso)
245 return secs * 1000
246
247
249 secs = iso_to_unix_time(iso)
250 t = time.mktime(time.localtime(secs))
251 return datetime.fromtimestamp(t)
252
253
255 """
256 Returns generator of all permutations of a
257
258 The following code is an in-place permutation of a given list, implemented
259 as a generator. Since it only returns references to the list, the list
260 should not be modified outside the generator. The solution is
261 non-recursive, so uses low memory. Work well also with multiple copies of
262 elements in the input list.
263
264 Retrieved from:
265 http://stackoverflow.com/questions/104420/ \
266 how-to-generate-all-permutations-of-a-list-in-python
267 """
268 a.sort()
269 yield list(a)
270 if len(a) <= 1:
271 return
272 first = 0
273 last = len(a)
274 while 1:
275 i = last - 1
276 while 1:
277 i = i - 1
278 if a[i] < a[i + 1]:
279 j = last - 1
280 while not (a[i] < a[j]):
281 j = j - 1
282
283 a[i], a[j] = a[j], a[i]
284 r = a[i + 1:last]
285 r.reverse()
286 a[i + 1:last] = r
287 yield list(a)
288 break
289 if i == first:
290 a.reverse()
291 return
292
293
302
303
305 """
306 Checks that all commands in the programs list exist. Returns
307 True if all commands exist and raises exception.CommandNotFound if not.
308 """
309 for prog in programs:
310 if not which(prog):
311 raise exception.CommandNotFound(prog)
312 return True
313
314
316 """
317 Returns the path to the program provided it exists and
318 is on the system's PATH
319
320 retrieved from code snippet by Jay:
321
322 http://stackoverflow.com/questions/377017/ \
323 test-if-executable-exists-in-python
324 """
325 def is_exe(fpath):
326 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
327 fpath, fname = os.path.split(program)
328 if fpath:
329 if is_exe(program):
330 return program
331 else:
332 for path in os.environ["PATH"].split(os.pathsep):
333 exe_file = os.path.join(path, program)
334 if is_exe(exe_file):
335 return exe_file
336
337
339 """
340 Constantly displays the last lines in filename
341 Similar to 'tail -f' unix command
342 """
343
344 file = open(filename, 'r')
345
346
347 st_results = os.stat(filename)
348 st_size = st_results[6]
349 file.seek(st_size)
350
351 while True:
352 where = file.tell()
353 line = file.readline()
354 if not line:
355 time.sleep(1)
356 file.seek(where)
357 continue
358 print line,
359
360
362 parts = v.split(suff)
363 if 2 != len(parts):
364 return v
365 version[4] = weight
366 version[5] = parts[1]
367 return parts[0]
368
369
371
372
373 """
374 Convert a Mozilla-style version string into a floating-point number
375 1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc
376 """
377 version = [
378 0, 0, 0, 0,
379 4,
380 0,
381 1
382 ]
383 parts = v.split("pre")
384 if 2 == len(parts):
385 version[6] = 0
386 v = parts[0]
387
388 v = v2fhelper(v, "a", version, 1)
389 v = v2fhelper(v, "b", version, 2)
390 v = v2fhelper(v, "rc", version, 3)
391
392 parts = v.split(".")[:4]
393 for (p, i) in zip(parts, range(len(parts))):
394 version[i] = p
395 ver = float(version[0])
396 ver += float(version[1]) / 100.
397 ver += float(version[2]) / 10000.
398 ver += float(version[3]) / 1000000.
399 ver += float(version[4]) / 100000000.
400 ver += float(version[5]) / 10000000000.
401 ver += float(version[6]) / 1000000000000.
402 return ver
403
404
406 """
407 Return True if ver1 > ver2 using semantics of comparing version
408 numbers
409 """
410 v1f = version_to_float(ver1)
411 v2f = version_to_float(ver2)
412 return v1f > v2f
413
414
416 assert program_version_greater("1", "0.9")
417 assert program_version_greater("0.0.0.2", "0.0.0.1")
418 assert program_version_greater("1.0", "0.9")
419 assert program_version_greater("2.0.1", "2.0.0")
420 assert program_version_greater("2.0.1", "2.0")
421 assert program_version_greater("2.0.1", "2")
422 assert program_version_greater("0.9.1", "0.9.0")
423 assert program_version_greater("0.9.2", "0.9.1")
424 assert program_version_greater("0.9.11", "0.9.2")
425 assert program_version_greater("0.9.12", "0.9.11")
426 assert program_version_greater("0.10", "0.9")
427 assert program_version_greater("2.0", "2.0b35")
428 assert program_version_greater("1.10.3", "1.10.3b3")
429 assert program_version_greater("88", "88a12")
430 assert program_version_greater("0.0.33", "0.0.33rc23")
431 assert program_version_greater("0.91.2", "0.91.1")
432 assert program_version_greater("0.9999", "0.91.1")
433 assert program_version_greater("0.9999", "0.92")
434 assert program_version_greater("0.91.10", "0.91.1")
435 assert program_version_greater("0.92", "0.91.11")
436 assert program_version_greater("0.92", "0.92b1")
437 assert program_version_greater("0.9999", "0.92b3")
438 print("All tests passed")
439
440
442 """
443 Convenience wrapper around inspect.getargspec
444
445 Returns a tuple whose first element is a list containing the names of all
446 required arguments and whose second element is a list containing the names
447 of all keyword (optional) arguments.
448 """
449 allargs, varargs, keywords, defaults = inspect.getargspec(func)
450 if 'self' in allargs:
451 allargs.remove('self')
452 nargs = len(allargs)
453 ndefaults = 0
454 if defaults:
455 ndefaults = len(defaults)
456 nrequired = nargs - ndefaults
457 args = allargs[:nrequired]
458 kwargs = allargs[nrequired:]
459 log.debug('nargs = %s' % nargs)
460 log.debug('ndefaults = %s' % ndefaults)
461 log.debug('nrequired = %s' % nrequired)
462 log.debug('args = %s' % args)
463 log.debug('kwargs = %s' % kwargs)
464 log.debug('defaults = %s' % str(defaults))
465 return args, kwargs
466
467
469 """
470 iterate through 'chunks' of a list. final chunk consists of remaining
471 elements if items does not divide len(ls) evenly.
472
473 items - size of 'chunks'
474 """
475 itms = []
476 for i, v in enumerate(ls):
477 if i >= items and i % items == 0:
478 yield itms
479 itms = [v]
480 else:
481 itms.append(v)
482 if itms:
483 yield itms
484