Package gridmap :: Module runner

Source Code for Module gridmap.runner

  1  # -*- coding: utf-8 -*- 
  2   
  3  # Written (W) 2008-2012 Christian Widmer 
  4  # Written (W) 2008-2010 Cheng Soon Ong 
  5  # Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org 
  6  # Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS 
  7   
  8  # This file is part of Grid Map. 
  9   
 10  # Grid Map is free software: you can redistribute it and/or modify 
 11  # it under the terms of the GNU General Public License as published by 
 12  # the Free Software Foundation, either version 3 of the License, or 
 13  # (at your option) any later version. 
 14   
 15  # Grid Map is distributed in the hope that it will be useful, 
 16  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 17  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 18  # GNU General Public License for more details. 
 19   
 20  # You should have received a copy of the GNU General Public License 
 21  # along with Grid Map.  If not, see <http://www.gnu.org/licenses/>. 
 22   
 23  """ 
 24  This module executes pickled jobs on the cluster. 
 25   
 26  @author: Christian Widmer 
 27  @author: Cheng Soon Ong 
 28  @author: Dan Blanchard (dblanchard@ets.org) 
 29  """ 
 30   
 31  from __future__ import absolute_import, print_function, unicode_literals 
 32   
 33  import argparse 
 34  import os 
 35  import sys 
 36   
 37  from redis import StrictRedis 
 38   
 39  from gridmap.data import clean_path, zload_db, zsave_db 
 40  from gridmap.job import REDIS_DB, REDIS_PORT 
 41   
 42   
43 -def _run_job(uniq_id, job_num, temp_dir, redis_host):
44 """ 45 Execute the pickled job and produce pickled output. 46 47 @param uniq_id: The unique suffix for the tables corresponding to this job 48 in the database. 49 @type uniq_id: C{basestring} 50 @param job_num: The index for this job's content in the job and output 51 tables. 52 @type job_num: C{int} 53 @param temp_dir: Local temporary directory for storing output for an 54 individual job. 55 @type temp_dir: C{basestring} 56 @param redis_host: Hostname of the database to connect to get the job data. 57 @type redis_host: C{basestring} 58 """ 59 # Connect to database 60 redis_server = StrictRedis(host=redis_host, port=REDIS_PORT, db=REDIS_DB) 61 62 print("Loading job...", end="", file=sys.stderr) 63 sys.stderr.flush() 64 try: 65 job = zload_db(redis_server, 'job_{0}'.format(uniq_id), job_num) 66 except Exception as detail: 67 job = None 68 print("FAILED", file=sys.stderr) 69 70 print("Writing exception to database for job {0}...".format(job_num), 71 end="", file=sys.stderr) 72 sys.stderr.flush() 73 zsave_db(detail, redis_server, 'output_{0}'.format(uniq_id), job_num) 74 print("done", file=sys.stderr) 75 else: 76 print("done", file=sys.stderr) 77 78 print("Running job...", end="", file=sys.stderr) 79 sys.stderr.flush() 80 job.execute() 81 print("done", file=sys.stderr) 82 83 print("Writing output to database for job {0}...".format(job_num), 84 end="", file=sys.stderr) 85 sys.stderr.flush() 86 zsave_db(job.ret, redis_server, 'output_{0}'.format(uniq_id), job_num) 87 print("done", file=sys.stderr) 88 89 #remove files 90 if job.cleanup: 91 log_stdout_fn = os.path.join(temp_dir, '{0}.o{1}'.format(job.name, 92 job.jobid)) 93 log_stderr_fn = os.path.join(temp_dir, '{0}.e{1}'.format(job.name, 94 job.jobid)) 95 96 try: 97 os.remove(log_stdout_fn) 98 os.remove(log_stderr_fn) 99 except OSError: 100 pass
101 102
103 -def _main():
104 """ 105 Parse the command line inputs and call _run_job 106 """ 107 108 # Get command line arguments 109 parser = argparse.ArgumentParser(description="This wrapper script will run \ 110 a pickled Python function on \ 111 some pickled data in a Redis\ 112 database, " + "and write the\ 113 results back to the database.\ 114 You almost never want to run\ 115 this yourself.", 116 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 117 conflict_handler='resolve') 118 parser.add_argument('uniq_id', 119 help='The unique suffix for the tables corresponding to\ 120 this job in the database.') 121 parser.add_argument('job_number', 122 help='Which job number should be run. Dictates which \ 123 input data is read from database and where output\ 124 data is stored.', 125 type=int) 126 parser.add_argument('module_dir', 127 help='Directory that contains module containing pickled\ 128 function. This will get added to PYTHONPATH \ 129 temporarily.') 130 parser.add_argument('temp_dir', 131 help='Directory that temporary output will be stored\ 132 in.') 133 parser.add_argument('redis_host', 134 help='The hostname of the server that where the Redis\ 135 database is.') 136 args = parser.parse_args() 137 138 print("Appended {0} to PYTHONPATH".format(args.module_dir), file=sys.stderr) 139 sys.path.append(clean_path(args.module_dir)) 140 141 # Process the database and get job started 142 _run_job(args.uniq_id, args.job_number, clean_path(args.temp_dir), 143 args.redis_host)
144 145 146 if __name__ == "__main__": 147 _main() 148