Package starcluster :: Package commands :: Module shell
[hide private]
[frames] | no frames]

Source Code for Module starcluster.commands.shell

  1  import os 
  2  import sys 
  3  import base64 
  4  import posixpath 
  5   
  6  import starcluster 
  7  from starcluster import utils 
  8  from starcluster import static 
  9  from starcluster.logger import log 
 10   
 11  from base import CmdBase 
 12   
 13   
14 -class CmdShell(CmdBase):
15 """ 16 shell 17 18 Load an interactive IPython shell configured for starcluster development 19 20 The following objects are automatically available at the prompt: 21 22 cfg - starcluster.config.StarClusterConfig instance 23 cm - starcluster.cluster.ClusterManager instance 24 ec2 - starcluster.awsutils.EasyEC2 instance 25 s3 - starcluster.awsutils.EasyS3 instance 26 27 All StarCluster modules are automatically imported in the IPython session 28 along with all StarCluster dependencies (e.g. boto, paramiko, etc.) 29 30 If the --ipcluster=CLUSTER (-p) is passed, the IPython session will be 31 automatically be configured to connect to the remote CLUSTER using 32 IPython's parallel interface (requires IPython 0.11+). In this mode you 33 will have the following additional objects available at the prompt: 34 35 ipcluster - starcluster.cluster.Cluster instance for the cluster 36 ipclient - IPython.parallel.Client instance for the cluster 37 ipview - IPython.parallel.client.view.DirectView for the cluster 38 39 Here's an example of how to run a parallel map across all nodes in the 40 cluster: 41 42 [~]> ipclient.ids 43 [0, 1, 2, 3] 44 [~]> res = ipview.map_async(lambda x: x**30, range(8)) 45 [~]> print res.get() 46 [0, 47 1, 48 1073741824, 49 205891132094649L, 50 1152921504606846976L, 51 931322574615478515625L, 52 221073919720733357899776L, 53 22539340290692258087863249L] 54 55 See IPython parallel docs for more details 56 (http://ipython.org/ipython-doc/stable/parallel) 57 """ 58 59 names = ['shell', 'sh'] 60
61 - def _add_to_known_hosts(self, node):
62 log.info("Configuring local known_hosts file") 63 user_home = os.path.expanduser('~') 64 khosts = os.path.join(user_home, '.ssh', 'known_hosts') 65 if not os.path.isfile(khosts): 66 log.warn("Unable to configure known_hosts: file does not exist") 67 return 68 contents = open(khosts).read() 69 if node.dns_name not in contents: 70 server_pkey = node.ssh.get_server_public_key() 71 khostsf = open(khosts, 'a') 72 if contents[-1] != '\n': 73 khostsf.write('\n') 74 name_entry = '%s,%s' % (node.dns_name, node.ip_address) 75 khostsf.write(' '.join([name_entry, server_pkey.get_name(), 76 base64.b64encode(str(server_pkey)), '\n'])) 77 khostsf.close()
78
79 - def addopts(self, parser):
80 parser.add_option("-p", "--ipcluster", dest="ipcluster", 81 action="store", type="string", default=None, 82 metavar="CLUSTER", help="configure a parallel " 83 "IPython session on CLUSTER")
84
85 - def execute(self, args):
86 local_ns = dict(cfg=self.cfg, ec2=self.ec2, s3=self.s3, cm=self.cm, 87 starcluster=starcluster) 88 if self.opts.ipcluster: 89 log.info("Loading parallel IPython library") 90 try: 91 from IPython.parallel import Client 92 except ImportError, e: 93 self.parser.error( 94 "Error loading parallel IPython:" 95 "\n\n%s\n\n" 96 "NOTE: IPython 0.11+ must be installed to use -p" % e) 97 tag = self.opts.ipcluster 98 cl = self.cm.get_cluster(tag) 99 region = cl.master_node.region.name 100 ipcluster_dir = os.path.join(static.STARCLUSTER_CFG_DIR, 101 'ipcluster') 102 local_json = os.path.join(ipcluster_dir, 103 "%s-%s.json" % (tag, region)) 104 if not os.path.exists(local_json): 105 user_home = cl.master_node.getpwnam(cl.cluster_user).pw_dir 106 profile_dir = posixpath.join(user_home, '.ipython', 107 'profile_default') 108 json = posixpath.join(profile_dir, 'security', 109 'ipcontroller-client.json') 110 if cl.master_node.ssh.isfile(json): 111 log.info("Fetching connector file from cluster...") 112 os.makedirs(ipcluster_dir) 113 cl.master_node.ssh.get(json, local_json) 114 else: 115 self.parser.error( 116 "IPython json file %s does not exist locally or on " 117 "the cluster. Make sure the ipcluster plugin has " 118 "been executed and completed successfully.") 119 key_location = cl.master_node.key_location 120 self._add_to_known_hosts(cl.master_node) 121 log.info("Loading parallel IPython client and view") 122 rc = Client(local_json, sshkey=key_location, packer='pickle') 123 local_ns['Client'] = Client 124 local_ns['ipcluster'] = cl 125 local_ns['ipclient'] = rc 126 local_ns['ipview'] = rc[:] 127 modules = [(starcluster.__name__ + '.' + module, module) 128 for module in starcluster.__all__] 129 modules += [('boto', 'boto'), ('paramiko', 'paramiko'), 130 ('workerpool', 'workerpool'), ('jinja2', 'jinja2')] 131 for fullname, modname in modules: 132 log.info('Importing module %s' % modname) 133 try: 134 __import__(fullname) 135 local_ns[modname] = sys.modules[fullname] 136 except ImportError, e: 137 log.error("Error loading module %s: %s" % (modname, e)) 138 utils.ipy_shell(local_ns=local_ns)
139