Package csb :: Module build
[frames] | no frames]

Source Code for Module csb.build

  1  """ 
  2  CSB build related tools and programs. 
  3   
  4  When executed as a program, this module will run the CSB Build Console and 
  5  build the source tree it belongs to. The source tree is added at the 
  6  B{beginning} of sys.path to make sure that all subsequent imports from the 
  7  Test and Doc consoles will import the right thing (think of multiple CSB 
  8  packages installed on the same server). 
  9   
 10  Here is how to build, test and package the whole project:: 
 11   
 12      $ hg clone https://hg.codeplex.com/csb CSB 
 13      $ CSB/csb/build.py -o <output directory> 
 14   
 15  The Console can also be imported and instantiated as a regular Python class. 
 16  In this case the Console again builds the source tree it is part of, but 
 17  sys.path will remain intact. Therefore, the Console will assume that all 
 18  modules currently in memory, as well as those that can be subsequently imported 
 19  by the Console itself, belong to the same CSB package. 
 20   
 21  @note: The CSB build services no longer support the option to build external 
 22         source trees. 
 23  @see: [CSB 0000038] 
 24  """ 
 25  from __future__ import print_function 
 26   
 27  import os 
 28  import sys 
 29  import getopt 
 30  import traceback 
 31           
 32  if os.path.basename(__file__) == '__init__.py': 
 33      PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 
 34  else: 
 35      PARENT = os.path.abspath(os.path.dirname(__file__)) 
 36   
 37  ROOT = 'csb' 
 38  SOURCETREE = os.path.abspath(os.path.join(PARENT, "..")) 
 39   
 40  if __name__ == '__main__': 
 41   
 42      # make sure "import io" imports the built in module, not csb.io 
 43      # io is required by tarfile     
 44      for path in sys.path: 
 45          if path.startswith(SOURCETREE): 
 46              sys.path.remove(path) 
 47               
 48      import io 
 49      assert hasattr(io, 'BufferedIOBase')    
 50               
 51      sys.path = [SOURCETREE] + sys.path 
 52   
 53   
 54  """ 
 55  It is now safe to import any modules   
 56  """ 
 57  import imp 
 58  import shutil 
 59  import tarfile 
 60   
 61  import csb 
 62   
 63  from abc import ABCMeta, abstractmethod 
 64  from csb.io import Shell 
65 66 67 -class BuildTypes(object):
68 """ 69 Enumeration of build types. 70 """ 71 72 SOURCE = 'source' 73 BINARY = 'binary' 74 75 _du = { SOURCE: 'sdist', BINARY: 'bdist' } 76 77 @staticmethod
78 - def get(key):
79 try: 80 return BuildTypes._du[key] 81 except KeyError: 82 raise ValueError('Unhandled build type: {0}'.format(key))
83
84 85 -class Console(object):
86 """ 87 CSB Build Bot. Run with -h for usage. 88 89 @param output: build output directory 90 @type output: str 91 @param verbosity: verbosity level 92 @type verbosity: int 93 94 @note: The build console automatically detects and builds the csb package 95 it belongs to. You cannot build a different source tree with it. 96 See the module documentation for more info. 97 """ 98 99 PROGRAM = __file__ 100 101 USAGE = r""" 102 CSB Build Console: build, test and package the entire csb project. 103 104 Usage: 105 python {program} -o output [-v verbosity] [-t type] [-h] 106 107 Options: 108 -o output Build output directory 109 -v verbosity Verbosity level, default is 1 110 -t type Build type: 111 source - build source code distribution (default) 112 binary - build executable 113 -h, --help Display this help 114 """ 115
116 - def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE):
117 118 self._input = None 119 self._output = None 120 self._temp = None 121 self._docs = None 122 self._apidocs = None 123 self._root = None 124 self._verbosity = None 125 self._type = buildtype 126 self._dist = BuildTypes.get(buildtype) 127 128 if os.path.join(SOURCETREE, ROOT) != PARENT: 129 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT)) 130 self._input = SOURCETREE 131 132 self.output = output 133 self.verbosity = verbosity
134 135 @property
136 - def input(self):
137 return self._input
138 139 @property
140 - def output(self):
141 return self._output
142 @output.setter
143 - def output(self, value):
144 #value = os.path.dirname(value) 145 self._output = os.path.abspath(value) 146 self._temp = os.path.join(self._output, 'build') 147 self._docs = os.path.join(self._temp, 'docs') 148 self._apidocs = os.path.join(self._docs, 'api') 149 self._root = os.path.join(self._temp, ROOT)
150 151 @property
152 - def verbosity(self):
153 return self._verbosity
154 @verbosity.setter
155 - def verbosity(self, value):
156 self._verbosity = int(value)
157
158 - def build(self):
159 """ 160 Run the console. 161 """ 162 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE)) 163 164 self._init() 165 v = self._revision() 166 self._doc(v) 167 self._test() 168 vn = self._package() 169 170 self.log('\n# Done ({0}).\n'.format(vn.full))
171
172 - def log(self, message, level=1, ending='\n'):
173 174 if self._verbosity >= level: 175 sys.stdout.write(message) 176 sys.stdout.write(ending) 177 sys.stdout.flush()
178
179 - def _init(self):
180 """ 181 Collect all required stuff in the output folder. 182 """ 183 self.log('# Preparing the file system...') 184 185 if not os.path.exists(self._output): 186 self.log('Creating output directory {0}'.format(self._output), level=2) 187 os.mkdir(self._output) 188 189 if os.path.exists(self._temp): 190 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2) 191 shutil.rmtree(self._temp) 192 193 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2) 194 shutil.copytree(self._input, self._temp) 195 196 if os.path.exists(self._apidocs): 197 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2) 198 shutil.rmtree(self._apidocs) 199 if not os.path.isdir(self._docs): 200 self.log('Creating docs directory {0}'.format(self._docs), level=2) 201 os.mkdir(self._docs) 202 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2) 203 os.mkdir(self._apidocs)
204
205 - def _revision(self):
206 """ 207 Write the actual revision number to L{ROOT}.__version__ 208 """ 209 self.log('\n# Setting the most recent Revision Number...') 210 root = os.path.join(self._root, '__init__.py') 211 212 self.log('Retrieving revision number from {0}'.format(root), level=2) 213 rh = MercurialHandler(root) 214 revision = rh.read().revision 215 216 self.log('Writing back revision number {0}'.format(revision), level=2) 217 version = rh.write(revision, root) 218 219 self.log(' This is {0}.__version__ {1}'.format(ROOT, version), level=1) 220 csb.__version__ = version 221 222 return version
223
224 - def _test(self):
225 """ 226 Run tests. Also make sure the current environment loads all modules from 227 the input folder. 228 """ 229 import csb.test 230 assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!' #@UndefinedVariable 231 232 from csb.test import unittest 233 234 newdata = os.path.join(self._temp, ROOT, 'test', 'data') 235 csb.test.Config.setDefaultDataRoot(newdata) 236 237 self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2) 238 csb.test.Config().ensureDataConsistency() 239 240 self.log('\n# Running the Test Console...') 241 242 builder = csb.test.AnyTestBuilder() 243 suite = builder.loadTests(ROOT + '.test.cases.*') 244 245 runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity) 246 result = runner.run(suite) 247 if result.wasSuccessful(): 248 self.log('\n Passed all unit tests') 249 else: 250 self.log('\n DID NOT PASS: The build might be broken')
251
252 - def _doc(self, version):
253 """ 254 Build documentation in the output folder. 255 """ 256 self.log('\n# Generating API documentation...') 257 try: 258 import epydoc.cli 259 except ImportError: 260 self.log('\n Skipped: epydoc is missing') 261 return 262 263 self.log('\n# Emulating ARGV for the Doc Builder...', level=2) 264 argv = sys.argv 265 sys.argv = ['epydoc', '--html', '-o', self._apidocs, 266 '--name', '{0} v{1}'.format(ROOT.upper(), version), 267 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases', 268 '--css', os.path.join(self._temp, 'epydoc.css'), 269 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning', 270 self._root] 271 272 if self._verbosity > 0: 273 sys.argv.append('-v') 274 275 try: 276 epydoc.cli.cli() 277 sys.exit(0) 278 except SystemExit as ex: 279 if ex.code is 0: 280 self.log('\n Passed all doc tests') 281 else: 282 if ex.code == 2: 283 self.log('\n DID NOT PASS: The docs might be broken') 284 else: 285 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex)) 286 287 self.log('\n# Restoring the previous ARGV...', level=2) 288 sys.argv = argv
289
290 - def _package(self):
291 """ 292 Make package. 293 """ 294 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2) 295 cwd = os.curdir 296 os.chdir(self._temp) 297 298 if self._verbosity > 1: 299 verbosity = '-v' 300 else: 301 verbosity = '-q' 302 argv = sys.argv 303 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output] 304 305 self.log('\n# Building {0} distribution...'.format(self._type)) 306 try: 307 setup = imp.load_source('setupcsb', 'setup.py') 308 d = setup.build() 309 version = setup.VERSION 310 package = d.dist_files[0][2] 311 312 if self._type == BuildTypes.BINARY: 313 self._strip_source(package) 314 315 except SystemExit as ex: 316 if ex.code is not 0: 317 package = 'FAIL' 318 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex)) 319 320 self.log('\n# Restoring the previous CWD and ARGV...', level=2) 321 os.chdir(cwd) 322 sys.argv = argv 323 324 self.log(' Packaged ' + package) 325 return version
326
327 - def _strip_source(self, package, source='*.py'):
328 """ 329 Delete plain text source code files from the package. 330 """ 331 cwd = os.getcwd() 332 333 try: 334 tmp = os.path.join(self.output, 'tmp') 335 os.mkdir(tmp) 336 337 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2) 338 os.chdir(tmp) 339 340 oldtar = tarfile.open(package, mode='r:gz') 341 oldtar.extractall(tmp) 342 oldtar.close() 343 344 newtar = tarfile.open(package, mode='w:gz') 345 346 try: 347 for i in os.walk('.'): 348 for fn in i[2]: 349 if fn.endswith('.py'): 350 module = os.path.join(i[0], fn); 351 if not os.path.isfile(module.replace('.py', '.pyc')): 352 raise ValueError('Missing bytecode for module {0}'.format(module)) 353 else: 354 os.remove(os.path.join(i[0], fn)) 355 356 for i in os.listdir('.'): 357 newtar.add(i) 358 finally: 359 newtar.close() 360 361 finally: 362 self.log('\n# Restoring the previous CWD...', level=2) 363 os.chdir(cwd) 364 if os.path.exists(tmp): 365 shutil.rmtree(tmp)
366 367 @staticmethod
368 - def exit(message=None, code=0, usage=True):
369 370 if message: 371 print(message) 372 if usage: 373 print(Console.USAGE.format(program=Console.PROGRAM)) 374 375 sys.exit(code)
376 377 @staticmethod
378 - def run(argv=None):
379 380 if argv is None: 381 argv = sys.argv[1:] 382 383 output = None 384 verb = 1 385 buildtype = BuildTypes.SOURCE 386 387 try: 388 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help']) 389 390 for option, value in options: 391 if option in('-h', '--help'): 392 Console.exit(message=None, code=0) 393 if option in('-o', '--output'): 394 if not os.path.isdir(value): 395 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3) 396 output = value 397 if option in('-v', '--verbosity'): 398 try: 399 verb = int(value) 400 except ValueError: 401 Console.exit(message='E: Verbosity must be an integer.', code=4) 402 if option in('-t', '--type'): 403 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]: 404 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5) 405 buildtype = value 406 except getopt.GetoptError as oe: 407 Console.exit(message='E: ' + str(oe), code=1) 408 409 if not output: 410 Console.exit(code=1, usage=True) 411 else: 412 try: 413 Console(output, verbosity=verb, buildtype=buildtype).build() 414 except Exception as ex: 415 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc()) 416 Console.exit(message=msg, code=99, usage=False)
417
418 419 -class RevisionError(RuntimeError):
420
421 - def __init__(self, msg, code, cmd):
422 423 super(RevisionError, self).__init__(msg) 424 self.code = code 425 self.cmd = cmd
426
427 -class RevisionHandler(object):
428 """ 429 Determines the current repository revision number of a working copy. 430 431 @param path: a local checkout path to be examined 432 @type path: str 433 @param sc: name of the source control program 434 @type sc: str 435 """ 436
437 - def __init__(self, path, sc):
438 439 self._path = None 440 self._sc = None 441 442 if os.path.exists(path): 443 self._path = path 444 else: 445 raise IOError('Path not found: {0}'.format(path)) 446 if Shell.run([sc, 'help']).code is 0: 447 self._sc = sc 448 else: 449 raise RevisionError('Source control binary probe failed', None, None)
450 451 @property
452 - def path(self):
453 return self._path
454 455 @property
456 - def sc(self):
457 return self._sc
458 459 @abstractmethod
460 - def read(self):
461 """ 462 Return the current revision information. 463 @rtype: L{RevisionInfo} 464 """ 465 pass
466
467 - def write(self, revision, sourcefile):
468 """ 469 Finalize the __version__ = major.minor.micro.{revision} tag. 470 Overwrite C{sourcefile} in place by substituting the {revision} macro. 471 472 @param revision: revision number to write to the source file. 473 @type revision: int 474 @param sourcefile: python source file with a __version__ tag, typically 475 "csb/__init__.py" 476 @type sourcefile: str 477 478 @return: sourcefile.__version__ 479 """ 480 content = open(sourcefile).readlines() 481 482 with open(sourcefile, 'w') as src: 483 for line in content: 484 if line.startswith('__version__'): 485 src.write(line.format(revision=revision)) 486 else: 487 src.write(line) 488 489 self._delcache(sourcefile) 490 return imp.load_source('____source', sourcefile).__version__
491
492 - def _run(self, cmd):
493 494 si = Shell.run(cmd) 495 if si.code > 0: 496 raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd) 497 498 return si.stdout.splitlines()
499
500 - def _delcache(self, sourcefile):
501 502 compiled = os.path.splitext(sourcefile)[0] + '.pyc' 503 if os.path.isfile(compiled): 504 os.remove(compiled) 505 506 pycache = os.path.join(os.path.dirname(compiled), '__pycache__') 507 if os.path.isdir(pycache): 508 shutil.rmtree(pycache)
509
510 -class SubversionHandler(RevisionHandler):
511
512 - def __init__(self, path, sc='svn'):
513 super(SubversionHandler, self).__init__(path, sc)
514
515 - def read(self):
516 517 cmd = '{0.sc} info {0.path} -R'.format(self) 518 maxrevision = None 519 520 for line in self._run(cmd): 521 if line.startswith('Revision:'): 522 rev = int(line[9:] .strip()) 523 if rev > maxrevision: 524 maxrevision = rev 525 526 if maxrevision is None: 527 raise RevisionError('No revision number found', code=0, cmd=cmd) 528 529 return RevisionInfo(self.path, maxrevision)
530
531 -class MercurialHandler(RevisionHandler):
532
533 - def __init__(self, path, sc='hg'):
534 535 if os.path.isfile(path): 536 path = os.path.dirname(path) 537 538 super(MercurialHandler, self).__init__(path, sc)
539
540 - def read(self):
541 542 wd = os.getcwd() 543 os.chdir(self.path) 544 545 try: 546 cmd = '{0.sc} log -r tip'.format(self) 547 548 revision = None 549 changeset = '' 550 551 for line in self._run(cmd): 552 if line.startswith('changeset:'): 553 items = line[10:].split(':') 554 revision = int(items[0]) 555 changeset = items[1].strip() 556 break 557 558 if revision is None: 559 raise RevisionError('No revision number found', code=0, cmd=cmd) 560 561 return RevisionInfo(self.path, revision, changeset) 562 563 finally: 564 os.chdir(wd)
565
566 -class RevisionInfo(object):
567
568 - def __init__(self, item, revision, id=None):
569 570 self.item = item 571 self.revision = revision 572 self.id = id
573
574 575 -def main():
576 Console.run()
577 578 579 if __name__ == '__main__': 580 581 main() 582