1
2 """
3 Utils module for StarCluster
4 """
5
6 import os
7 import re
8 import time
9 import types
10 import calendar
11 import urlparse
12 from datetime import datetime
13
14 from starcluster import iptools
15 from starcluster import exception
16 from starcluster.logger import log
17
18
20 """
21 Subclass of dict that allows read-only attribute-like access to
22 dictionary key/values
23 """
25 try:
26 return self.__getitem__(name)
27 except KeyError:
28 return super(AttributeDict, self).__getattribute__(name)
29
30
32 """
33 Decorator for printing execution time (in mins) of a function
34 Optionally takes a user-friendly msg as argument. This msg will
35 appear in the sentence "[msg] took XXX mins". If no msg is specified,
36 msg will default to the decorated function's name. e.g:
37
38 @print_timing
39 def myfunc():
40 print 'hi'
41 >>> myfunc()
42 hi
43 myfunc took 0.000 mins
44
45 @print_timing('My function')
46 def myfunc():
47 print 'hi'
48 >>> myfunc()
49 hi
50 My function took 0.000 mins
51 """
52 if type(msg) == types.FunctionType:
53 func = msg
54
55 def wrap_f(*arg, **kargs):
56 """Raw timing function """
57 time1 = time.time()
58 res = func(*arg, **kargs)
59 time2 = time.time()
60 prefix = func.func_name
61 log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
62 return res
63 return wrap_f
64
65 def wrap(func):
66 def wrap_f(*arg, **kargs):
67 """Raw timing function """
68 time1 = time.time()
69 res = func(*arg, **kargs)
70 time2 = time.time()
71 prefix = msg
72 log.info('%s took %0.3f mins' % (prefix, (time2 - time1) / 60.0))
73 return res
74 return wrap_f
75 return wrap
76
77
79 """
80 Checks that dev matches the following regular expression:
81 /dev/sd[a-z]$
82 """
83 regex = re.compile('/dev/sd[a-z]$')
84 try:
85 return regex.match(dev) is not None
86 except TypeError:
87 return False
88
89
91 """
92 Checks that part matches the following regular expression:
93 /dev/sd[a-z][1-9][0-9]?$
94 """
95 regex = re.compile('/dev/sd[a-z][1-9][0-9]?$')
96 try:
97 return regex.match(part) is not None
98 except TypeError:
99 return False
100
101
103 """
104 Check if bucket_name is a valid S3 bucket name (as defined by the AWS
105 docs):
106
107 1. 3 <= len(bucket_name) <= 255
108 2. all chars one of: a-z 0-9 . _ -
109 3. first char one of: a-z 0-9
110 4. name must not be a valid ip
111 """
112 regex = re.compile('[a-z0-9][a-z0-9\._-]{2,254}$')
113 if not regex.match(bucket_name):
114 return False
115 if iptools.validate_ip(bucket_name):
116 return False
117 return True
118
119
121 """
122 Check if image_name is a valid AWS image name (as defined by the AWS docs)
123
124 1. 3<= len(image_name) <=128
125 2. all chars one of: a-z A-Z 0-9 ( ) . - / _
126 """
127 regex = re.compile('[\w\(\)\.\-\/_]{3,128}$')
128 try:
129 return regex.match(image_name) is not None
130 except TypeError:
131 return False
132
133
135 """
136 Returns command to execute python script as a one-line python program
137
138 e.g.
139
140 import os
141 script = '''
142 import os
143 print os.path.exists('hi')
144 '''
145 os.system(make_one_liner(script))
146
147 Will print out:
148
149 <module 'os' from ...>
150 False
151 """
152 return 'python -c "%s"' % script.strip().replace('\n', ';')
153
154
156 """
157 Returns True if the provided string is a valid url
158 """
159 try:
160 parts = urlparse.urlparse(url)
161 scheme = parts[0]
162 netloc = parts[1]
163 if scheme and netloc:
164 return True
165 else:
166 return False
167 except:
168 return False
169
170
172 """
173 Returns True if provided time can be parsed in iso format
174 to a datetime tuple
175 """
176 try:
177 iso_to_datetime_tuple(iso)
178 return True
179 except ValueError:
180 return False
181
182
184 """
185 Converts an iso time string to a datetime tuple
186 """
187
188 iso = iso.split('.')[0]
189 try:
190 return datetime.strptime(iso, "%Y-%m-%dT%H:%M:%S")
191 except AttributeError:
192
193 return datetime(*time.strptime(iso, "%Y-%m-%dT%H:%M:%S")[:6])
194
195
197 """
198 Converts a datetime tuple to iso time string
199 """
200 iso = datetime.strftime(tup, "%Y-%m-%dT%H:%M:%S")
201 return iso
202
203
205 ptime = iso_to_localtime_tuple(past_time)
206 now = datetime.now()
207 delta = now - ptime
208 return time.strftime("%H:%M:%S", time.gmtime(delta.seconds))
209
210
212 dtup = iso_to_datetime_tuple(iso)
213 secs = calendar.timegm(dtup.timetuple())
214 t = time.mktime(time.localtime(secs))
215 return datetime.fromtimestamp(t)
216
217 try:
218 import IPython.Shell
219 ipy_shell = IPython.Shell.IPShellEmbed(argv=[])
220 except ImportError:
221
223 log.error("Unable to load IPython.")
224 log.error("Please check that IPython is installed and working.")
225 log.error("If not, you can install it via: easy_install ipython")
226
227
229 """
230 Returns generator of all permutations of a
231
232 The following code is an in-place permutation of a given list, implemented
233 as a generator. Since it only returns references to the list, the list
234 should not be modified outside the generator. The solution is
235 non-recursive, so uses low memory. Work well also with multiple copies of
236 elements in the input list.
237
238 Retrieved from:
239 http://stackoverflow.com/questions/104420/ \
240 how-to-generate-all-permutations-of-a-list-in-python
241 """
242 a.sort()
243 yield list(a)
244 if len(a) <= 1:
245 return
246 first = 0
247 last = len(a)
248 while 1:
249 i = last - 1
250 while 1:
251 i = i - 1
252 if a[i] < a[i + 1]:
253 j = last - 1
254 while not (a[i] < a[j]):
255 j = j - 1
256
257 a[i], a[j] = a[j], a[i]
258 r = a[i + 1:last]
259 r.reverse()
260 a[i + 1:last] = r
261 yield list(a)
262 break
263 if i == first:
264 a.reverse()
265 return
266
267
276
277
279 """
280 Checks that all commands in the programs list exist. Returns
281 True if all commands exist and raises exception.CommandNotFound if not.
282 """
283 for prog in programs:
284 if not which(prog):
285 raise exception.CommandNotFound(prog)
286 return True
287
288
290 """
291 Returns the path to the program provided it exists and
292 is on the system's PATH
293
294 retrieved from code snippet by Jay:
295
296 http://stackoverflow.com/questions/377017/ \
297 test-if-executable-exists-in-python
298 """
299 def is_exe(fpath):
300 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
301 fpath, fname = os.path.split(program)
302 if fpath:
303 if is_exe(program):
304 return program
305 else:
306 for path in os.environ["PATH"].split(os.pathsep):
307 exe_file = os.path.join(path, program)
308 if is_exe(exe_file):
309 return exe_file
310
311
313 """
314 Constantly displays the last lines in filename
315 Similar to 'tail -f' unix command
316 """
317
318 file = open(filename, 'r')
319
320
321 st_results = os.stat(filename)
322 st_size = st_results[6]
323 file.seek(st_size)
324
325 while True:
326 where = file.tell()
327 line = file.readline()
328 if not line:
329 time.sleep(1)
330 file.seek(where)
331 continue
332 print line,
333
334
336 parts = v.split(suff)
337 if 2 != len(parts):
338 return v
339 version[4] = weight
340 version[5] = parts[1]
341 return parts[0]
342
343
345
346
347 """
348 Convert a Mozilla-style version string into a floating-point number
349 1.2.3.4, 1.2a5, 2.3.4b1pre, 3.0rc2, etc
350 """
351 version = [
352 0, 0, 0, 0,
353 4,
354 0,
355 1
356 ]
357 parts = v.split("pre")
358 if 2 == len(parts):
359 version[6] = 0
360 v = parts[0]
361
362 v = v2fhelper(v, "a", version, 1)
363 v = v2fhelper(v, "b", version, 2)
364 v = v2fhelper(v, "rc", version, 3)
365
366 parts = v.split(".")[:4]
367 for (p, i) in zip(parts, range(len(parts))):
368 version[i] = p
369 ver = float(version[0])
370 ver += float(version[1]) / 100.
371 ver += float(version[2]) / 10000.
372 ver += float(version[3]) / 1000000.
373 ver += float(version[4]) / 100000000.
374 ver += float(version[5]) / 10000000000.
375 ver += float(version[6]) / 1000000000000.
376 return ver
377
378
380 """
381 Return True if ver1 > ver2 using semantics of comparing version
382 numbers
383 """
384 v1f = version_to_float(ver1)
385 v2f = version_to_float(ver2)
386 return v1f > v2f
387
388
390 assert program_version_greater("1", "0.9")
391 assert program_version_greater("0.0.0.2", "0.0.0.1")
392 assert program_version_greater("1.0", "0.9")
393 assert program_version_greater("2.0.1", "2.0.0")
394 assert program_version_greater("2.0.1", "2.0")
395 assert program_version_greater("2.0.1", "2")
396 assert program_version_greater("0.9.1", "0.9.0")
397 assert program_version_greater("0.9.2", "0.9.1")
398 assert program_version_greater("0.9.11", "0.9.2")
399 assert program_version_greater("0.9.12", "0.9.11")
400 assert program_version_greater("0.10", "0.9")
401 assert program_version_greater("2.0", "2.0b35")
402 assert program_version_greater("1.10.3", "1.10.3b3")
403 assert program_version_greater("88", "88a12")
404 assert program_version_greater("0.0.33", "0.0.33rc23")
405 assert program_version_greater("0.91.2", "0.91.1")
406 assert program_version_greater("0.9999", "0.91.1")
407 assert program_version_greater("0.9999", "0.92")
408 assert program_version_greater("0.91.10", "0.91.1")
409 assert program_version_greater("0.92", "0.91.11")
410 assert program_version_greater("0.92", "0.92b1")
411 assert program_version_greater("0.9999", "0.92b3")
412 print("All tests passed")
413