Package ziggy :: Package hdmc :: Package hdfs :: Module hdfs_access
[hide private]
[frames] | no frames]

Source Code for Module ziggy.hdmc.hdfs.hdfs_access

 1  ''' 
 2  Created on May 17, 2010 
 3   
 4  @author: dwmclary 
 5  ''' 
 6   
 7  from . import hdfs_config as config 
 8  import subprocess 
 9   
10   
11 -def run(args):
12 '''Generic runner for HDFS access''' 13 p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr = subprocess.PIPE) 14 r = p.communicate() 15 result = {"stdout":r[0], "stderr":r[1]} 16 return result
17 18
19 -def cat(filename):
20 '''Runs HDFS cat on a filename or glob passed as a string 21 Returns the result as a string read from STOUT''' 22 args = [config.hadoop, "dfs", "-cat", filename] 23 return run(args)
24 25
26 -def ls(filename):
27 '''Runs HDFS ls on a directory or glob passed as a string 28 Returns the result as a string read from STOUT''' 29 args = [config.hadoop, "dfs", "-ls", filename] 30 return run(args)
31
32 -def tail(filename):
33 args = [config.hadoop, "dfs", "-tail", filename] 34 return run(args)
35
36 -def mv(filename1, filename2):
37 args = [config.hadoop, "dfs", "-mv", filename1, filename2] 38 return run(args)
39
40 -def rm(filename):
41 '''Runs HDFS rmr on a filename or glob passed as a string 42 Returns the result as a string read from STOUT''' 43 args = [config.hadoop, "dfs", "-rmr", filename] 44 return run(args)
45
46 -def delete(filename):
47 '''Runs HDFS rmr on a filename or glob passed as a string 48 Returns the result as a string read from STOUT''' 49 args = [config.hadoop, "dfs", "-rm", filename] 50 return run(args)
51
52 -def mkdir(dirname):
53 '''Runs HDFS mkdir on a directory name as a string 54 Returns the result as a string read from STOUT''' 55 args = [config.hadoop, "dfs", "-mkdir", dirname] 56 return run(args)
57
58 -def get_merge(pattern, dst):
59 '''Runs HDFS rmr on a filename or glob passed as a string 60 Returns the result as a string read from STOUT''' 61 args = [config.hadoop, "dfs", "-getmerge", pattern, dst] 62 return run(args)
63
64 -def mv(src, dst):
65 '''Runs HDFS mv: move src HDFS file to dst 66 Returns the result as a string read from STOUT''' 67 args = [config.hadoop, "dfs", "-mv", src, dst] 68 return run(args)
69 70 71
72 -def copyToHDFS(filename, hdfs_filename):
73 args = [config.hadoop, "dfs", "-put", filename, hdfs_filename] 74 return run(args)
75
76 -def copyFromHDFS(hdfs_filename, filename):
77 args = [config.hadoop, "dfs", "-get", hdfs_filename, filename] 78 return run(args)
79