1 import os
2 import sys
3 import base64
4 import posixpath
5
6 import starcluster
7 from starcluster import utils
8 from starcluster import static
9 from starcluster.logger import log
10
11 from base import CmdBase
12
13
15 """
16 shell
17
18 Load an interactive IPython shell configured for starcluster development
19
20 The following objects are automatically available at the prompt:
21
22 cfg - starcluster.config.StarClusterConfig instance
23 cm - starcluster.cluster.ClusterManager instance
24 ec2 - starcluster.awsutils.EasyEC2 instance
25 s3 - starcluster.awsutils.EasyS3 instance
26
27 All StarCluster modules are automatically imported in the IPython session
28 along with all StarCluster dependencies (e.g. boto, paramiko, etc.)
29
30 If the --ipcluster=CLUSTER (-p) is passed, the IPython session will be
31 automatically be configured to connect to the remote CLUSTER using
32 IPython's parallel interface (requires IPython 0.11+). In this mode you
33 will have the following additional objects available at the prompt:
34
35 ipcluster - starcluster.cluster.Cluster instance for the cluster
36 ipclient - IPython.parallel.Client instance for the cluster
37 ipview - IPython.parallel.client.view.DirectView for the cluster
38
39 Here's an example of how to run a parallel map across all nodes in the
40 cluster:
41
42 [~]> ipclient.ids
43 [0, 1, 2, 3]
44 [~]> res = ipview.map_async(lambda x: x**30, range(8))
45 [~]> print res.get()
46 [0,
47 1,
48 1073741824,
49 205891132094649L,
50 1152921504606846976L,
51 931322574615478515625L,
52 221073919720733357899776L,
53 22539340290692258087863249L]
54
55 See IPython parallel docs for more details
56 (http://ipython.org/ipython-doc/stable/parallel)
57 """
58
59 names = ['shell', 'sh']
60
62 log.info("Configuring local known_hosts file")
63 user_home = os.path.expanduser('~')
64 khosts = os.path.join(user_home, '.ssh', 'known_hosts')
65 if not os.path.isfile(khosts):
66 log.warn("Unable to configure known_hosts: file does not exist")
67 return
68 contents = open(khosts).read()
69 if node.dns_name not in contents:
70 server_pkey = node.ssh.get_server_public_key()
71 khostsf = open(khosts, 'a')
72 if contents[-1] != '\n':
73 khostsf.write('\n')
74 name_entry = '%s,%s' % (node.dns_name, node.ip_address)
75 khostsf.write(' '.join([name_entry, server_pkey.get_name(),
76 base64.b64encode(str(server_pkey)), '\n']))
77 khostsf.close()
78
80 parser.add_option("-p", "--ipcluster", dest="ipcluster",
81 action="store", type="string", default=None,
82 metavar="CLUSTER", help="configure a parallel "
83 "IPython session on CLUSTER")
84
86 local_ns = dict(cfg=self.cfg, ec2=self.ec2, s3=self.s3, cm=self.cm,
87 starcluster=starcluster)
88 if self.opts.ipcluster:
89 log.info("Loading parallel IPython library")
90 try:
91 from IPython.parallel import Client
92 except ImportError, e:
93 self.parser.error(
94 "Error loading parallel IPython:"
95 "\n\n%s\n\n"
96 "NOTE: IPython 0.11+ must be installed to use -p" % e)
97 tag = self.opts.ipcluster
98 cl = self.cm.get_cluster(tag)
99 region = cl.master_node.region.name
100 ipcluster_dir = os.path.join(static.STARCLUSTER_CFG_DIR,
101 'ipcluster')
102 local_json = os.path.join(ipcluster_dir,
103 "%s-%s.json" % (tag, region))
104 if not os.path.exists(local_json):
105 user_home = cl.master_node.getpwnam(cl.cluster_user).pw_dir
106 profile_dir = posixpath.join(user_home, '.ipython',
107 'profile_default')
108 json = posixpath.join(profile_dir, 'security',
109 'ipcontroller-client.json')
110 if cl.master_node.ssh.isfile(json):
111 log.info("Fetching connector file from cluster...")
112 os.makedirs(ipcluster_dir)
113 cl.master_node.ssh.get(json, local_json)
114 else:
115 self.parser.error(
116 "IPython json file %s does not exist locally or on "
117 "the cluster. Make sure the ipcluster plugin has "
118 "been executed and completed successfully.")
119 key_location = cl.master_node.key_location
120 self._add_to_known_hosts(cl.master_node)
121 log.info("Loading parallel IPython client and view")
122 rc = Client(local_json, sshkey=key_location, packer='pickle')
123 local_ns['Client'] = Client
124 local_ns['ipcluster'] = cl
125 local_ns['ipclient'] = rc
126 local_ns['ipview'] = rc[:]
127 modules = [(starcluster.__name__ + '.' + module, module)
128 for module in starcluster.__all__]
129 modules += [('boto', 'boto'), ('paramiko', 'paramiko'),
130 ('workerpool', 'workerpool'), ('jinja2', 'jinja2')]
131 for fullname, modname in modules:
132 log.info('Importing module %s' % modname)
133 try:
134 __import__(fullname)
135 local_ns[modname] = sys.modules[fullname]
136 except ImportError, e:
137 log.error("Error loading module %s: %s" % (modname, e))
138 utils.ipy_shell(local_ns=local_ns)
139