Package starcluster :: Module clustersetup
[hide private]
[frames] | no frames]

Source Code for Module starcluster.clustersetup

  1  #!/usr/bin/env python 
  2   
  3  """ 
  4  clustersetup.py 
  5  """ 
  6  import re 
  7  import posixpath 
  8   
  9  from starcluster import threadpool 
 10  from starcluster.utils import print_timing 
 11  from starcluster.templates import sge 
 12  from starcluster.logger import log 
13 14 15 -class ClusterSetup(object):
16 """ 17 ClusterSetup Interface 18 19 This is the base class for all StarCluster plugins. A plugin should 20 implement at least one if not all of these methods. 21 """
22 - def __init__(self, *args, **kwargs):
23 pass
24
25 - def on_add_node(self, node, nodes, master, user, user_shell, volumes):
26 """ 27 This methods gets executed after a node has been added to the cluster 28 """ 29 raise NotImplementedError('on_add_node method not implemented')
30
31 - def on_remove_node(self, node, nodes, master, user, user_shell, volumes):
32 """ 33 This method gets executed before a node is about to be removed from the 34 cluster 35 """ 36 raise NotImplementedError('on_remove_node method not implemented')
37
38 - def on_restart(self, nodes, master, user, user_shell, volumes):
39 """ 40 This method gets executed before restart the cluster 41 """ 42 raise NotImplementedError('on_restart method not implemented')
43
44 - def on_shutdown(self, nodes, master, user, user_shell, volumes):
45 """ 46 This method gets executed before shutting down the cluster 47 """ 48 raise NotImplementedError('on_shutdown method not implemented')
49
50 - def run(self, nodes, master, user, user_shell, volumes):
51 """ 52 Run this plugin's setup routines 53 54 This method gets executed after the default cluster setup routines have 55 been performed 56 """ 57 raise NotImplementedError('run method not implemented')
58
59 60 -class DefaultClusterSetup(ClusterSetup):
61 """ 62 Default ClusterSetup implementation for StarCluster 63 """
64 - def __init__(self, disable_queue=False, disable_threads=False, 65 num_threads=20):
66 self._nodes = None 67 self._master = None 68 self._user = None 69 self._user_shell = None 70 self._volumes = None 71 self._disable_queue = disable_queue 72 self._disable_threads = disable_threads 73 self._num_threads = num_threads 74 self._pool = None
75 76 @property
77 - def pool(self):
78 if not self._pool: 79 self._pool = threadpool.get_thread_pool(self._num_threads, 80 self._disable_threads) 81 return self._pool
82 83 @property
84 - def nodes(self):
85 return filter(lambda x: not x.is_master(), self._nodes)
86 87 @property
88 - def running_nodes(self):
89 return filter(lambda x: x.state in ['running'], self._nodes)
90
91 - def _setup_hostnames(self, nodes=None):
92 """ 93 Set each node's hostname to their alias. 94 """ 95 nodes = nodes or self._nodes 96 log.info("Configuring hostnames...") 97 for node in nodes: 98 self.pool.simple_job(node.set_hostname, (), jobid=node.alias) 99 self.pool.wait(numtasks=len(nodes))
100
101 - def _setup_cluster_user(self):
102 """ 103 Create cluster user on all StarCluster nodes 104 105 This command takes care to examine existing folders in /home 106 and set the new cluster_user's uid/gid accordingly. This is necessary 107 for the case of EBS volumes containing /home with large amounts of data 108 in them. It's much less expensive in this case to set the uid/gid of 109 the new user to be the existing uid/gid of the dir in EBS rather than 110 chowning potentially terabytes of data. 111 """ 112 mconn = self._master.ssh 113 home_folder = '/home/%s' % self._user 114 first_uid = 1000 115 uid, gid = first_uid, first_uid 116 umap = self._master.get_user_map(key_by_uid=True) 117 if mconn.path_exists(home_folder): 118 # get /home/user's owner/group uid and create 119 # user with that uid/gid 120 s = mconn.stat(home_folder) 121 uid = s.st_uid 122 gid = s.st_gid 123 else: 124 # get highest uid/gid of dirs in /home/*, 125 # increment by 1 and create user with that uid/gid 126 uid_db = {} 127 files = mconn.ls('/home') 128 for file in files: 129 if mconn.isdir(file): 130 f = mconn.stat(file) 131 uid_db[f.st_uid] = (file, f.st_gid) 132 if uid_db.keys(): 133 max_uid = max(uid_db.keys()) 134 max_gid = uid_db[max_uid][1] 135 uid, gid = max_uid + 1, max_gid + 1 136 # make sure the newly selected uid/gid is >= 1000 137 uid = max(uid, first_uid) 138 gid = max(gid, first_uid) 139 # make sure newly selected uid is not already in /etc/passwd 140 while umap.get(uid): 141 uid += 1 142 gid += 1 143 log.info("Creating cluster user: %s (uid: %d, gid: %d)" % (self._user, 144 uid, gid)) 145 self._add_user_to_nodes(uid, gid, self._nodes)
146
147 - def _add_user_to_node(self, uid, gid, node):
148 existing_user = node.getpwuid(uid) 149 if existing_user: 150 username = existing_user.pw_name 151 if username != self._user: 152 msg = ("user %s exists on %s with same uid/gid as " 153 "cluster user %s...removing user %s") 154 log.debug( 155 msg % (username, node.alias, self._user, username)) 156 node.remove_user(username) 157 node.add_user(self._user, uid, gid, self._user_shell) 158 log.debug("user %s exists on node %s, no action" % \ 159 (self._user, node.alias)) 160 else: 161 log.debug("user %s does not exist, creating..." % self._user) 162 node.add_user(self._user, uid, gid, self._user_shell)
163
164 - def _add_user_to_nodes(self, uid, gid, nodes=None):
165 nodes = nodes or self._nodes 166 for node in nodes: 167 self.pool.simple_job(self._add_user_to_node, (uid, gid, node), 168 jobid=node.alias) 169 self.pool.wait(numtasks=len(nodes))
170
171 - def _setup_scratch_on_node(self, node):
172 nconn = node.ssh 173 user_scratch = '/mnt/%s' % self._user 174 if not nconn.path_exists(user_scratch): 175 nconn.mkdir(user_scratch) 176 nconn.execute('chown -R %(user)s:%(user)s /mnt/%(user)s' % \ 177 {'user': self._user}) 178 scratch = '/scratch' 179 if not nconn.path_exists(scratch): 180 nconn.mkdir(scratch) 181 if not nconn.path_exists(posixpath.join(scratch, self._user)): 182 nconn.execute('ln -s %s %s' % (user_scratch, scratch))
183
184 - def _setup_scratch(self, nodes=None):
185 """ Configure scratch space on all StarCluster nodes """ 186 log.info("Configuring scratch space for user: %s" % self._user) 187 nodes = nodes or self._nodes 188 for node in nodes: 189 self.pool.simple_job(self._setup_scratch_on_node, (node,), 190 jobid=node.alias) 191 self.pool.wait(numtasks=len(nodes))
192
193 - def _setup_etc_hosts(self, nodes=None):
194 """ Configure /etc/hosts on all StarCluster nodes""" 195 log.info("Configuring /etc/hosts on each node") 196 nodes = nodes or self._nodes 197 for node in nodes: 198 self.pool.simple_job(node.add_to_etc_hosts, (nodes, ), 199 jobid=node.alias) 200 self.pool.wait(numtasks=len(nodes))
201
202 - def _setup_passwordless_ssh(self, nodes=None):
203 """ 204 Properly configure passwordless ssh for root and CLUSTER_USER on all 205 StarCluster nodes 206 """ 207 log.info("Configuring passwordless ssh for root") 208 master = self._master 209 nodes = nodes or self.nodes 210 master.generate_key_for_user('root', auth_new_key=True, 211 auth_conn_key=True) 212 master.enable_passwordless_ssh('root', nodes) 213 # generate public/private keys, authorized_keys, and known_hosts files 214 # for cluster_user once on master node...NFS takes care of the rest 215 log.info("Configuring passwordless ssh for %s" % self._user) 216 master.generate_key_for_user(self._user, auth_new_key=True, 217 auth_conn_key=True) 218 master.add_to_known_hosts(self._user, nodes)
219
220 - def _setup_ebs_volumes(self):
221 """ 222 Mount EBS volumes, if specified in ~/.starcluster/config to /home 223 """ 224 # setup /etc/fstab on master to use block device if specified 225 master = self._master 226 devs = master.ssh.ls('/dev') 227 for vol in self._volumes: 228 vol = self._volumes[vol] 229 vol_id = vol.get("volume_id") 230 mount_path = vol.get('mount_path') 231 device = vol.get("device") 232 volume_partition = vol.get('partition') 233 if not (vol_id and device and mount_path): 234 log.error("missing required settings for vol %s" % vol) 235 continue 236 dev_exists = master.ssh.path_exists(device) 237 if not dev_exists and device.startswith('/dev/sd'): 238 # check for "correct" device in unpatched kernels 239 device = device.replace('/dev/sd', '/dev/xvd') 240 dev_exists = master.ssh.path_exists(device) 241 if not dev_exists: 242 log.warn("Cannot find device %s for volume %s" % 243 (device, vol_id)) 244 log.warn("Not mounting %s on %s" % (vol_id, mount_path)) 245 log.warn("This usually means there was a problem " 246 "attaching the EBS volume to the master node") 247 continue 248 if not volume_partition: 249 partitions = filter(lambda x: x.startswith(device), devs) 250 if len(partitions) == 1: 251 volume_partition = device 252 elif len(partitions) == 2: 253 volume_partition = device + '1' 254 else: 255 log.error( 256 "volume has more than one partition, please specify " 257 "which partition to use (e.g. partition=0, " 258 "partition=1, etc) in the volume's config") 259 continue 260 elif not master.ssh.path_exists(volume_partition): 261 log.warn("Cannot find partition %s on volume %s" % \ 262 (volume_partition, vol_id)) 263 log.warn("Not mounting %s on %s" % (vol_id, 264 mount_path)) 265 log.warn("This either means that the volume has not " + \ 266 "been partitioned or that the partition" + \ 267 "specified does not exist on the volume") 268 continue 269 log.info("Mounting EBS volume %s on %s..." % (vol_id, mount_path)) 270 mount_map = self._master.get_mount_map() 271 dev = mount_map.get(volume_partition) 272 if dev: 273 path, fstype, options = dev 274 if path != mount_path: 275 log.error("Volume %s is mounted on %s, not on %s" % \ 276 (vol_id, path, mount_path)) 277 else: 278 log.info( 279 "Volume %s already mounted on %s...skipping" % \ 280 (vol_id, mount_path)) 281 continue 282 self._master.mount_device(volume_partition, mount_path)
283
284 - def _get_nfs_export_paths(self):
285 export_paths = ['/home'] 286 if not self._disable_queue: 287 export_paths.append('/opt/sge6') 288 for vol in self._volumes: 289 vol = self._volumes[vol] 290 mount_path = vol.get('mount_path') 291 if not mount_path in export_paths: 292 export_paths.append(mount_path) 293 return export_paths
294
295 - def _mount_nfs_shares(self, nodes):
296 # setup /etc/fstab and mount each nfs share on each node 297 master = self._master 298 export_paths = self._get_nfs_export_paths() 299 for node in nodes: 300 mount_map = node.get_mount_map() 301 mount_paths = [] 302 for path in export_paths: 303 network_device = "%s:%s" % (master.alias, path) 304 if network_device in mount_map: 305 mount_path, type, options = mount_map.get(network_device) 306 log.debug(('nfs share %s already mounted to %s on ' + \ 307 'node %s, skipping...') % \ 308 (network_device, mount_path, node.alias)) 309 else: 310 mount_paths.append(path) 311 self.pool.simple_job(node.mount_nfs_shares, (master, mount_paths), 312 jobid=node.alias) 313 self.pool.wait(numtasks=len(nodes))
314 315 @print_timing("Setting up NFS")
316 - def _setup_nfs(self, nodes=None, start_server=True):
317 """ 318 Share /home, /opt/sge6, and all EBS mount paths via NFS to all nodes 319 """ 320 log.info("Configuring NFS...") 321 master = self._master 322 if not self._disable_queue and not master.ssh.isdir('/opt/sge6'): 323 # copy fresh sge installation files to /opt/sge6 324 master.ssh.execute('cp -r /opt/sge6-fresh /opt/sge6') 325 master.ssh.execute('chown -R %(user)s:%(user)s /opt/sge6' % \ 326 {'user': self._user}) 327 # setup /etc/exports and start nfsd on master node 328 nodes = nodes or self.nodes 329 export_paths = self._get_nfs_export_paths() 330 if start_server: 331 master.start_nfs_server() 332 master.export_fs_to_nodes(nodes, export_paths) 333 self._mount_nfs_shares(nodes)
334
335 - def _create_sge_pe(self, name="orte", nodes=None, queue="all.q"):
336 """ 337 Create or update an SGE parallel environment 338 339 name - name of parallel environment 340 nodes - list of nodes to include in the parallel environment 341 (default: all) 342 queue - configure queue to use the new parallel environment 343 """ 344 mssh = self._master.ssh 345 pe_exists = mssh.get_status('qconf -sp %s' % name, source_profile=True) 346 pe_exists = pe_exists == 0 347 if not pe_exists: 348 log.info("Creating SGE parallel environment '%s'" % name) 349 else: 350 log.info("Updating SGE parallel environment '%s'" % name) 351 # iterate through each machine and count the number of processors 352 nodes = nodes or self._nodes 353 num_processors = sum(self.pool.map(lambda n: n.num_processors, nodes)) 354 penv = mssh.remote_file("/tmp/pe.txt") 355 print >> penv, sge.sge_pe_template % (name, num_processors) 356 penv.close() 357 if not pe_exists: 358 mssh.execute("qconf -Ap %s" % penv.name, source_profile=True) 359 else: 360 mssh.execute("qconf -Mp %s" % penv.name, source_profile=True) 361 if queue: 362 log.info("Adding parallel environment '%s' to queue '%s'" % 363 (name, queue)) 364 mssh.execute('qconf -mattr queue pe_list "%s" %s' % (name, queue), 365 source_profile=True)
366
367 - def _setup_sge(self):
368 """ 369 Install Sun Grid Engine with a default parallel 370 environment on StarCluster 371 """ 372 # generate /etc/profile.d/sge.sh for each node 373 for node in self._nodes: 374 conn = node.ssh 375 conn.execute('pkill -9 sge', ignore_exit_status=True) 376 conn.execute('rm /etc/init.d/sge*', ignore_exit_status=True) 377 sge_profile = conn.remote_file("/etc/profile.d/sge.sh") 378 arch = conn.execute("/opt/sge6/util/arch")[0] 379 print >> sge_profile, sge.sgeprofile_template % {'arch': arch} 380 sge_profile.close() 381 # setup sge auto install file 382 master = self._master 383 default_cell = '/opt/sge6/default' 384 if master.ssh.isdir(default_cell): 385 log.info("Removing previous SGE installation...") 386 master.ssh.execute('rm -rf %s' % default_cell) 387 master.ssh.execute('exportfs -fr') 388 mconn = master.ssh 389 admin_list = ' '.join(map(lambda n: n.alias, self._nodes)) 390 exec_list = admin_list 391 submit_list = admin_list 392 ec2_sge_conf = mconn.remote_file("/opt/sge6/ec2_sge.conf") 393 # TODO: add sge section to config values for some of the below 394 conf = sge.sgeinstall_template % (admin_list, exec_list, submit_list) 395 print >> ec2_sge_conf, conf 396 ec2_sge_conf.close() 397 # installs sge in /opt/sge6 and starts qmaster/schedd on master node 398 log.info("Installing Sun Grid Engine...") 399 mconn.execute('cd /opt/sge6 && TERM=rxvt ./inst_sge -m -x -noremote ' 400 '-auto ./ec2_sge.conf', silent=True, only_printable=True) 401 # set all.q shell to bash 402 mconn.execute('qconf -mattr queue shell "/bin/bash" all.q', 403 source_profile=True) 404 for node in self.nodes: 405 self._add_sge_administrative_host(node) 406 self._add_sge_submit_host(node) 407 self.pool.simple_job(self._add_to_sge, (node,), jobid=node.alias) 408 self.pool.wait(numtasks=len(self.nodes)) 409 self._create_sge_pe()
410
411 - def run(self, nodes, master, user, user_shell, volumes):
412 """Start cluster configuration""" 413 try: 414 self._nodes = nodes 415 self._master = master 416 self._user = user 417 self._user_shell = user_shell 418 self._volumes = volumes 419 self._setup_hostnames() 420 self._setup_ebs_volumes() 421 self._setup_cluster_user() 422 self._setup_scratch() 423 self._setup_etc_hosts() 424 self._setup_nfs() 425 self._setup_passwordless_ssh() 426 if not self._disable_queue: 427 self._setup_sge() 428 finally: 429 self.pool.shutdown()
430
431 - def _remove_from_etc_hosts(self, node):
432 nodes = filter(lambda x: x.id != node.id, self.running_nodes) 433 for n in nodes: 434 n.remove_from_etc_hosts([node])
435
436 - def _remove_nfs_exports(self, node):
437 self._master.stop_exporting_fs_to_nodes([node])
438
439 - def _remove_from_known_hosts(self, node):
440 nodes = filter(lambda x: x.id != node.id, self.running_nodes) 441 for n in nodes: 442 n.remove_from_known_hosts('root', [node]) 443 n.remove_from_known_hosts(self._user, [node])
444
445 - def _remove_from_sge(self, node):
446 master = self._master 447 master.ssh.execute('qconf -shgrp @allhosts > /tmp/allhosts', 448 source_profile=True) 449 hgrp_file = master.ssh.remote_file('/tmp/allhosts', 'r') 450 contents = hgrp_file.read().splitlines() 451 hgrp_file.close() 452 c = [] 453 for line in contents: 454 line = line.replace(node.alias, '') 455 c.append(line) 456 hgrp_file = master.ssh.remote_file('/tmp/allhosts_new', 'w') 457 hgrp_file.writelines('\n'.join(c)) 458 hgrp_file.close() 459 master.ssh.execute('qconf -Mhgrp /tmp/allhosts_new', 460 source_profile=True) 461 master.ssh.execute('qconf -sq all.q > /tmp/allq', source_profile=True) 462 allq_file = master.ssh.remote_file('/tmp/allq', 'r') 463 contents = allq_file.read() 464 allq_file.close() 465 c = [l.strip() for l in contents.splitlines()] 466 s = [] 467 allq = [] 468 for l in c: 469 if l.startswith('slots') or l.startswith('['): 470 s.append(l) 471 else: 472 allq.append(l) 473 regex = re.compile(r"\[%s=\d+\],?" % node.alias) 474 slots = [] 475 for line in s: 476 line = line.replace('\\', '') 477 slots.append(regex.sub('', line)) 478 allq.append(''.join(slots)) 479 f = master.ssh.remote_file('/tmp/allq_new', 'w') 480 allq[-1] = allq[-1].strip() 481 if allq[-1].endswith(','): 482 allq[-1] = allq[-1][:-1] 483 f.write('\n'.join(allq)) 484 f.close() 485 master.ssh.execute('qconf -Mq /tmp/allq_new', source_profile=True) 486 master.ssh.execute('qconf -dconf %s' % node.alias, source_profile=True) 487 master.ssh.execute('qconf -de %s' % node.alias, source_profile=True) 488 node.ssh.execute('pkill -9 sge_execd') 489 nodes = filter(lambda n: n.alias != node.alias, self._nodes) 490 self._create_sge_pe(nodes=nodes)
491
492 - def on_remove_node(self, node, nodes, master, user, user_shell, volumes):
493 self._nodes = nodes 494 self._master = master 495 self._user = user 496 self._user_shell = user_shell 497 self._volumes = volumes 498 log.info("Removing node %s (%s)..." % (node.alias, node.id)) 499 if not self._disable_queue: 500 log.info("Removing %s from SGE" % node.alias) 501 self._remove_from_sge(node) 502 log.info("Removing %s from known_hosts files" % node.alias) 503 self._remove_from_known_hosts(node) 504 log.info("Removing %s from /etc/hosts" % node.alias) 505 self._remove_from_etc_hosts(node) 506 log.info("Removing %s from NFS" % node.alias) 507 self._remove_nfs_exports(node)
508
509 - def _create_user(self, node):
510 user = self._master.getpwnam(self._user) 511 uid, gid = user.pw_uid, user.pw_gid 512 self._add_user_to_nodes(uid, gid, nodes=[node])
513
514 - def _add_sge_submit_host(self, node):
515 mssh = self._master.ssh 516 mssh.execute('qconf -as %s' % node.alias, source_profile=True)
517
518 - def _add_sge_administrative_host(self, node):
519 mssh = self._master.ssh 520 mssh.execute('qconf -ah %s' % node.alias, source_profile=True)
521
522 - def _add_to_sge(self, node):
523 # generate /etc/profile.d/sge.sh 524 sge_profile = node.ssh.remote_file("/etc/profile.d/sge.sh") 525 arch = node.ssh.execute("/opt/sge6/util/arch")[0] 526 print >> sge_profile, sge.sgeprofile_template % {'arch': arch} 527 sge_profile.close() 528 node.ssh.execute('cd /opt/sge6 && TERM=rxvt ./inst_sge -x -noremote ' 529 '-auto ./ec2_sge.conf')
530
531 - def on_add_node(self, node, nodes, master, user, user_shell, volumes):
532 self._nodes = nodes 533 self._master = master 534 self._user = user 535 self._user_shell = user_shell 536 self._volumes = volumes 537 self._setup_hostnames(nodes=[node]) 538 self._setup_etc_hosts(nodes) 539 self._setup_nfs(nodes=[node], start_server=False) 540 self._create_user(node) 541 self._setup_scratch(nodes=[node]) 542 self._setup_passwordless_ssh(nodes=[node]) 543 if not self._disable_queue: 544 log.info("Adding %s to SGE" % node.alias) 545 self._add_sge_administrative_host(node) 546 self._add_sge_submit_host(node) 547 self._add_to_sge(node) 548 self._create_sge_pe()
549