1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 """
24 This modules provides all of the data-related function for gridmap.
25
26 @author: Christian Widmer
27 @author: Cheng Soon Ong
28 @author: Dan Blanchard (dblanchard@ets.org)
29 """
30
31 from __future__ import absolute_import, print_function, unicode_literals
32
33 import bz2
34 try:
35 import cPickle as pickle
36 except ImportError:
37 import pickle
38 import re
39 from time import sleep
40
41
42
43 MAX_TRIES = 50
44 SLEEP_TIME = 3
45
46
48 '''
49 Replace all weird SAN paths with normal paths. This is really
50 ETS-specific, but shouldn't harm anyone else.
51 '''
52
53 path = re.sub(r'/\.automount/\w+/SAN/NLP/(\w+)-(dynamic|static)',
54 r'/home/nlp-\1/\2', path)
55 path = re.sub(r'/\.automount/[^/]+/SAN/Research/HomeResearch',
56 '/home/research', path)
57 return path
58
59
60 -def zsave_db(obj, redis_server, prefix, job_num):
61 """
62 Saves an object/function as bz2-compressed pickled data in a Redis database
63
64 @param obj: The object/function to store.
65 @type obj: C{object} or C{function}
66 @param redis_server: An open connection to the database
67 @type redis_server: C{StrictRedis}
68 @param prefix: The prefix to use for the key for this data.
69 @type prefix: C{basestring}
70 @param job_num: The ID of the job this data is for.
71 @type job_num: C{int}
72 """
73
74
75 pickled_data = bz2.compress(pickle.dumps(obj, pickle.HIGHEST_PROTOCOL), 9)
76
77
78 redis_server.set('{0}_{1}'.format(prefix, job_num), pickled_data)
79
80
81 -def zload_db(redis_server, prefix, job_num):
82 """
83 Loads bz2-compressed pickled object from a Redis database
84
85 @param redis_server: An open connection to the database
86 @type redis_server: C{StrictRedis}
87 @param prefix: The prefix to use for the key for this data.
88 @type prefix: C{basestring}
89 @param job_num: The ID of the job this data is for.
90 @type job_num: C{int}
91 """
92 attempt = 0
93 pickled_data = None
94 while pickled_data is None and attempt < MAX_TRIES:
95 pickled_data = redis_server.get('{0}_{1}'.format(prefix, job_num))
96 attempt += 1
97 sleep(SLEEP_TIME)
98 return pickle.loads(bz2.decompress(pickled_data))
99