Package gridmap :: Module data

Source Code for Module gridmap.data

  1  # -*- coding: utf-8 -*- 
  2   
  3  # Written (W) 2008-2012 Christian Widmer 
  4  # Written (W) 2008-2010 Cheng Soon Ong 
  5  # Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org 
  6  # Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS 
  7   
  8  # This file is part of Grid Map. 
  9   
 10  # Grid Map is free software: you can redistribute it and/or modify 
 11  # it under the terms of the GNU General Public License as published by 
 12  # the Free Software Foundation, either version 3 of the License, or 
 13  # (at your option) any later version. 
 14   
 15  # Grid Map is distributed in the hope that it will be useful, 
 16  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 17  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 18  # GNU General Public License for more details. 
 19   
 20  # You should have received a copy of the GNU General Public License 
 21  # along with Grid Map.  If not, see <http://www.gnu.org/licenses/>. 
 22   
 23  """ 
 24  This modules provides all of the data-related function for gridmap. 
 25   
 26  @author: Christian Widmer 
 27  @author: Cheng Soon Ong 
 28  @author: Dan Blanchard (dblanchard@ets.org) 
 29  """ 
 30   
 31  from __future__ import absolute_import, print_function, unicode_literals 
 32   
 33  import bz2 
 34  try: 
 35      import cPickle as pickle  # For Python 2.x 
 36  except ImportError: 
 37      import pickle 
 38  import re 
 39  from time import sleep 
 40   
 41   
 42  #### Global settings #### 
 43  MAX_TRIES = 50 
 44  SLEEP_TIME = 3 
 45   
 46   
47 -def clean_path(path):
48 ''' 49 Replace all weird SAN paths with normal paths. This is really 50 ETS-specific, but shouldn't harm anyone else. 51 ''' 52 53 path = re.sub(r'/\.automount/\w+/SAN/NLP/(\w+)-(dynamic|static)', 54 r'/home/nlp-\1/\2', path) 55 path = re.sub(r'/\.automount/[^/]+/SAN/Research/HomeResearch', 56 '/home/research', path) 57 return path
58 59
60 -def zsave_db(obj, redis_server, prefix, job_num):
61 """ 62 Saves an object/function as bz2-compressed pickled data in a Redis database 63 64 @param obj: The object/function to store. 65 @type obj: C{object} or C{function} 66 @param redis_server: An open connection to the database 67 @type redis_server: C{StrictRedis} 68 @param prefix: The prefix to use for the key for this data. 69 @type prefix: C{basestring} 70 @param job_num: The ID of the job this data is for. 71 @type job_num: C{int} 72 """ 73 74 # Pickle the obj 75 pickled_data = bz2.compress(pickle.dumps(obj, pickle.HIGHEST_PROTOCOL), 9) 76 77 # Insert the pickled data into the database 78 redis_server.set('{0}_{1}'.format(prefix, job_num), pickled_data)
79 80
81 -def zload_db(redis_server, prefix, job_num):
82 """ 83 Loads bz2-compressed pickled object from a Redis database 84 85 @param redis_server: An open connection to the database 86 @type redis_server: C{StrictRedis} 87 @param prefix: The prefix to use for the key for this data. 88 @type prefix: C{basestring} 89 @param job_num: The ID of the job this data is for. 90 @type job_num: C{int} 91 """ 92 attempt = 0 93 pickled_data = None 94 while pickled_data is None and attempt < MAX_TRIES: 95 pickled_data = redis_server.get('{0}_{1}'.format(prefix, job_num)) 96 attempt += 1 97 sleep(SLEEP_TIME) 98 return pickle.loads(bz2.decompress(pickled_data))
99