Package gridmap :: Module data

Source Code for Module gridmap.data

  1  # -*- coding: utf-8 -*- 
  2   
  3  # Written (W) 2008-2012 Christian Widmer 
  4  # Written (W) 2008-2010 Cheng Soon Ong 
  5  # Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org 
  6  # Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS 
  7   
  8  # This file is part of Grid Map. 
  9   
 10  # Grid Map is free software: you can redistribute it and/or modify 
 11  # it under the terms of the GNU General Public License as published by 
 12  # the Free Software Foundation, either version 3 of the License, or 
 13  # (at your option) any later version. 
 14   
 15  # Grid Map is distributed in the hope that it will be useful, 
 16  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 17  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 18  # GNU General Public License for more details. 
 19   
 20  # You should have received a copy of the GNU General Public License 
 21  # along with Grid Map.  If not, see <http://www.gnu.org/licenses/>. 
 22   
 23  """ 
 24  This modules provides all of the data-related function for gridmap. 
 25   
 26  @author: Christian Widmer 
 27  @author: Cheng Soon Ong 
 28  @author: Dan Blanchard (dblanchard@ets.org) 
 29  """ 
 30   
 31  from __future__ import absolute_import, print_function, unicode_literals 
 32   
 33  import bz2 
 34  import os 
 35  try: 
 36      import cPickle as pickle  # For Python 2.x 
 37  except ImportError: 
 38      import pickle 
 39  import re 
 40  from time import sleep 
 41   
 42   
 43  #### Global settings #### 
 44  MAX_TRIES = int(os.getenv('GRID_MAP_MAX_TRIES', '10')) 
 45  SLEEP_TIME = int(os.getenv('GRID_MAP_SLEEP_TIME', '3')) 
 46   
 47   
48 -def clean_path(path):
49 ''' 50 Replace all weird SAN paths with normal paths. This is really 51 ETS-specific, but shouldn't harm anyone else. 52 ''' 53 54 path = re.sub(r'/\.automount/\w+/SAN/NLP/(\w+)-(dynamic|static)', 55 r'/home/nlp-\1/\2', path) 56 path = re.sub(r'/\.automount/[^/]+/SAN/Research/HomeResearch', 57 '/home/research', path) 58 return path
59 60
61 -def zsave_db(obj, redis_server, prefix, job_num):
62 """ 63 Saves an object/function as bz2-compressed pickled data in a Redis database 64 65 @param obj: The object/function to store. 66 @type obj: C{object} or C{function} 67 @param redis_server: An open connection to the database 68 @type redis_server: C{StrictRedis} 69 @param prefix: The prefix to use for the key for this data. 70 @type prefix: C{basestring} 71 @param job_num: The ID of the job this data is for. 72 @type job_num: C{int} 73 """ 74 75 # Pickle the obj 76 pickled_data = bz2.compress(pickle.dumps(obj, pickle.HIGHEST_PROTOCOL), 9) 77 78 # Insert the pickled data into the database 79 redis_server.set('{0}_{1}'.format(prefix, job_num), pickled_data)
80 81
82 -def zload_db(redis_server, prefix, job_num):
83 """ 84 Loads bz2-compressed pickled object from a Redis database 85 86 @param redis_server: An open connection to the database 87 @type redis_server: C{StrictRedis} 88 @param prefix: The prefix to use for the key for this data. 89 @type prefix: C{basestring} 90 @param job_num: The ID of the job this data is for. 91 @type job_num: C{int} 92 """ 93 attempt = 0 94 pickled_data = None 95 while pickled_data is None and attempt < MAX_TRIES: 96 pickled_data = redis_server.get('{0}_{1}'.format(prefix, job_num)) 97 attempt += 1 98 sleep(SLEEP_TIME) 99 return pickle.loads(bz2.decompress(pickled_data))
100