Package csb :: Module build
[frames] | no frames]

Source Code for Module csb.build

  1  """ 
  2  CSB build related tools and programs. 
  3   
  4  When executed as a program, this module will run the CSB Build Console and 
  5  build the source tree it belongs to. The source tree is added at the 
  6  B{beginning} of sys.path to make sure that all subsequent imports from the 
  7  Test and Doc consoles will import the right thing (think of multiple CSB 
  8  packages installed on the same server). 
  9   
 10  Here is how to build, test and package the whole project:: 
 11   
 12      $ svn checkout https://svn.tuebingen.mpg.de/agbs/projects/CSB 
 13      $ CSB/trunk/csb/build.py -o <output directory> 
 14   
 15  The Console can also be imported and instantiated as a regular Python class. 
 16  In this case the Console again builds the source tree it is part of, but 
 17  sys.path will remain intact. Therefore, the Console will assume that all 
 18  modules currently in memory, as well as those that can be subsequently imported 
 19  by the Console itself, belong to the same CSB package. 
 20   
 21  @note: The CSB build services no longer support the option to build external 
 22         source trees. 
 23  @see: [CSB 0000038] 
 24  """ 
 25  from __future__ import print_function 
 26   
 27  import os 
 28  import sys 
 29  import getopt 
 30  import traceback 
 31           
 32  if os.path.basename(__file__) == '__init__.py': 
 33      PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 
 34  else: 
 35      PARENT = os.path.abspath(os.path.dirname(__file__)) 
 36   
 37  ROOT = 'csb' 
 38  SOURCETREE = os.path.abspath(os.path.join(PARENT, "..")) 
 39   
 40  if __name__ == '__main__': 
 41   
 42      # make sure "import io" imports the built in module, not csb.io 
 43      # io is required by tarfile     
 44      for path in sys.path: 
 45          if path.startswith(SOURCETREE): 
 46              sys.path.remove(path) 
 47               
 48      import io 
 49      assert hasattr(io, 'BufferedIOBase')    
 50               
 51      sys.path = [SOURCETREE] + sys.path 
 52   
 53   
 54  """ 
 55  It is now safe to import any modules   
 56  """ 
 57  import imp 
 58  import shutil 
 59  import tarfile 
 60   
 61  import csb 
 62   
 63  from csb.io import Shell 
64 65 66 -class BuildTypes(object):
67 """ 68 Enumeration of build types. 69 """ 70 71 SOURCE = 'source' 72 BINARY = 'binary' 73 74 _du = { SOURCE: 'sdist', BINARY: 'bdist' } 75 76 @staticmethod
77 - def get(key):
78 try: 79 return BuildTypes._du[key] 80 except KeyError: 81 raise ValueError('Unhandled build type: {0}'.format(key))
82
83 84 -class Console(object):
85 """ 86 CSB Build Bot. Run with -h for usage. 87 88 @param output: build output directory 89 @type output: str 90 @param verbosity: verbosity level 91 @type verbosity: int 92 93 @note: The build console automatically detects and builds the csb package 94 it belongs to. You cannot build a different source tree with it. 95 See the module documentation for more info. 96 """ 97 98 PROGRAM = __file__ 99 100 USAGE = r""" 101 CSB Build Console: build, test and package the entire csb project. 102 103 Usage: 104 python {program} -o output [-v verbosity] [-t type] [-h] 105 106 Options: 107 -o output Build output directory 108 -v verbosity Verbosity level, default is 1 109 -t type Build type: 110 source - build source code distribution (default) 111 binary - build executable 112 -h, --help Display this help 113 """ 114
115 - def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE):
116 117 self._input = None 118 self._output = None 119 self._temp = None 120 self._docs = None 121 self._apidocs = None 122 self._root = None 123 self._verbosity = None 124 self._type = buildtype 125 self._dist = BuildTypes.get(buildtype) 126 127 if os.path.join(SOURCETREE, ROOT) != PARENT: 128 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT)) 129 self._input = SOURCETREE 130 131 self.output = output 132 self.verbosity = verbosity
133 134 @property
135 - def input(self):
136 return self._input
137 138 @property
139 - def output(self):
140 return self._output
141 @output.setter
142 - def output(self, value):
143 #value = os.path.dirname(value) 144 self._output = os.path.abspath(value) 145 self._temp = os.path.join(self._output, 'build') 146 self._docs = os.path.join(self._temp, 'docs') 147 self._apidocs = os.path.join(self._docs, 'api') 148 self._root = os.path.join(self._temp, ROOT)
149 150 @property
151 - def verbosity(self):
152 return self._verbosity
153 @verbosity.setter
154 - def verbosity(self, value):
155 self._verbosity = int(value)
156
157 - def build(self):
158 """ 159 Run the console. 160 """ 161 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE)) 162 163 self._init() 164 v = self._revision() 165 self._doc(v) 166 self._test() 167 vn = self._package() 168 169 self.log('\n# Done ({0}).\n'.format(vn.full))
170
171 - def log(self, message, level=1, ending='\n'):
172 173 if self._verbosity >= level: 174 sys.stdout.write(message) 175 sys.stdout.write(ending) 176 sys.stdout.flush()
177
178 - def _init(self):
179 """ 180 Collect all required stuff in the output folder. 181 """ 182 self.log('# Preparing the file system...') 183 184 if not os.path.exists(self._output): 185 self.log('Creating output directory {0}'.format(self._output), level=2) 186 os.mkdir(self._output) 187 188 if os.path.exists(self._temp): 189 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2) 190 shutil.rmtree(self._temp) 191 192 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2) 193 shutil.copytree(self._input, self._temp) 194 195 if os.path.exists(self._apidocs): 196 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2) 197 shutil.rmtree(self._apidocs) 198 if not os.path.isdir(self._docs): 199 self.log('Creating docs directory {0}'.format(self._docs), level=2) 200 os.mkdir(self._docs) 201 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2) 202 os.mkdir(self._apidocs)
203
204 - def _revision(self):
205 """ 206 Write the actual revision number to L{ROOT}.__version__ 207 """ 208 self.log('\n# Setting the most recent Revision Number...') 209 root = os.path.join(self._root, '__init__.py') 210 211 self.log('Retrieving revision number from {0}'.format(root), level=2) 212 rh = RevisionHandler(root) 213 revision = rh.read().maxrevision 214 215 self.log('Writing back revision number {0}'.format(revision), level=2) 216 version = rh.write(revision, root) 217 218 self.log(' This is {0}.__version__ {1}'.format(ROOT, version), level=1) 219 csb.__version__ = version 220 221 return version
222
223 - def _test(self):
224 """ 225 Run tests. Also make sure the current environment loads all modules from 226 the input folder. 227 """ 228 import csb.test 229 assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!' #@UndefinedVariable 230 231 from csb.test import unittest 232 233 newdata = os.path.join(self._temp, ROOT, 'test', 'data') 234 csb.test.Config.setDefaultDataRoot(newdata) 235 236 self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2) 237 csb.test.Config().ensureDataConsistency() 238 239 self.log('\n# Running the Test Console...') 240 241 builder = csb.test.AnyTestBuilder() 242 suite = builder.loadTests(ROOT + '.test.cases.*') 243 244 runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity) 245 result = runner.run(suite) 246 if result.wasSuccessful(): 247 self.log('\n Passed all unit tests') 248 else: 249 self.log('\n DID NOT PASS: The build might be broken')
250
251 - def _doc(self, version):
252 """ 253 Build documentation in the output folder. 254 """ 255 self.log('\n# Generating API documentation...') 256 try: 257 import epydoc.cli 258 except ImportError: 259 self.log('\n Skipped: epydoc is missing') 260 return 261 262 self.log('\n# Emulating ARGV for the Doc Builder...', level=2) 263 argv = sys.argv 264 sys.argv = ['epydoc', '--html', '-o', self._apidocs, 265 '--name', '{0} v{1}'.format(ROOT.upper(), version), 266 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases', 267 '--css', os.path.join(self._temp, 'epydoc.css'), 268 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning', 269 self._root] 270 271 if self._verbosity > 0: 272 sys.argv.append('-v') 273 274 try: 275 epydoc.cli.cli() 276 sys.exit(0) 277 except SystemExit as ex: 278 if ex.code is 0: 279 self.log('\n Passed all doc tests') 280 else: 281 if ex.code == 2: 282 self.log('\n DID NOT PASS: The docs might be broken') 283 else: 284 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex)) 285 286 self.log('\n# Restoring the previous ARGV...', level=2) 287 sys.argv = argv
288
289 - def _package(self):
290 """ 291 Make package. 292 """ 293 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2) 294 cwd = os.curdir 295 os.chdir(self._temp) 296 297 if self._verbosity > 1: 298 verbosity = '-v' 299 else: 300 verbosity = '-q' 301 argv = sys.argv 302 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output] 303 304 self.log('\n# Building {0} distribution...'.format(self._type)) 305 try: 306 setup = imp.load_source('setupcsb', 'setup.py') 307 d = setup.build() 308 version = setup.VERSION 309 package = d.dist_files[0][2] 310 311 if self._type == BuildTypes.BINARY: 312 self._strip_source(package) 313 314 except SystemExit as ex: 315 if ex.code is not 0: 316 package = 'FAIL' 317 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex)) 318 319 self.log('\n# Restoring the previous CWD and ARGV...', level=2) 320 os.chdir(cwd) 321 sys.argv = argv 322 323 self.log(' Packaged ' + package) 324 return version
325
326 - def _strip_source(self, package, source='*.py'):
327 """ 328 Delete plain text source code files from the package. 329 """ 330 cwd = os.getcwd() 331 332 try: 333 tmp = os.path.join(self.output, 'tmp') 334 os.mkdir(tmp) 335 336 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2) 337 os.chdir(tmp) 338 339 oldtar = tarfile.open(package, mode='r:gz') 340 oldtar.extractall(tmp) 341 oldtar.close() 342 343 newtar = tarfile.open(package, mode='w:gz') 344 345 try: 346 for i in os.walk('.'): 347 for fn in i[2]: 348 if fn.endswith('.py'): 349 module = os.path.join(i[0], fn); 350 if not os.path.isfile(module.replace('.py', '.pyc')): 351 raise ValueError('Missing bytecode for module {0}'.format(module)) 352 else: 353 os.remove(os.path.join(i[0], fn)) 354 355 for i in os.listdir('.'): 356 newtar.add(i) 357 finally: 358 newtar.close() 359 360 finally: 361 self.log('\n# Restoring the previous CWD...', level=2) 362 os.chdir(cwd) 363 if os.path.exists(tmp): 364 shutil.rmtree(tmp)
365 366 @staticmethod
367 - def exit(message=None, code=0, usage=True):
368 369 if message: 370 print(message) 371 if usage: 372 print(Console.USAGE.format(program=Console.PROGRAM)) 373 374 sys.exit(code)
375 376 @staticmethod
377 - def run(argv=None):
378 379 if argv is None: 380 argv = sys.argv[1:] 381 382 output = None 383 verb = 1 384 buildtype = BuildTypes.SOURCE 385 386 try: 387 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help']) 388 389 for option, value in options: 390 if option in('-h', '--help'): 391 Console.exit(message=None, code=0) 392 if option in('-o', '--output'): 393 if not os.path.isdir(value): 394 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3) 395 output = value 396 if option in('-v', '--verbosity'): 397 try: 398 verb = int(value) 399 except ValueError: 400 Console.exit(message='E: Verbosity must be an integer.', code=4) 401 if option in('-t', '--type'): 402 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]: 403 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5) 404 buildtype = value 405 except getopt.GetoptError as oe: 406 Console.exit(message='E: ' + str(oe), code=1) 407 408 if not output: 409 Console.exit(code=1, usage=True) 410 else: 411 try: 412 Console(output, verbosity=verb, buildtype=buildtype).build() 413 except Exception as ex: 414 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc()) 415 Console.exit(message=msg, code=99, usage=False)
416
417 418 -class RevisionError(RuntimeError):
419
420 - def __init__(self, msg, code, cmd):
421 422 super(RevisionError, self).__init__(msg) 423 self.code = code 424 self.cmd = cmd
425
426 -class RevisionHandler(object):
427 """ 428 Determines the current SVN revision number of a working copy. 429 430 @param path: a local checkout path to be examined 431 @type path: str 432 @param svn: name of the svn program 433 @type svn: str 434 """ 435
436 - def __init__(self, path, svn='svn'):
437 438 self.path = None 439 self.svn = None 440 441 if os.path.exists(path): 442 self.path = path 443 else: 444 raise IOError('Path not found: {0}'.format(path)) 445 if Shell.run([svn, 'help']).code is 0: 446 self.svn = svn 447 else: 448 raise RevisionError('SVN probe failed', None, None)
449
450 - def read(self):
451 """ 452 Return the current revision information. 453 @rtype: L{RevisionInfo} 454 455 @todo: we can easily extend the svn output parser to grab more attributes, 456 say URL, author, etc. RevisionInfo would also has to be extended 457 """ 458 cmd = '{0.svn} info {0.path}'.format(self) 459 revision = 0 460 maxrevision = 0 461 462 for line in self._run(cmd): 463 if line.startswith('Revision:'): 464 revision = int(line[9:] .strip()) 465 break 466 467 for line in self._run(cmd + ' -R'): 468 if line.startswith('Revision:'): 469 rev = int(line[9:] .strip()) 470 if rev > maxrevision: 471 maxrevision = rev 472 473 return RevisionInfo(self.path, revision, maxrevision)
474
475 - def write(self, revision, sourcefile):
476 """ 477 Finalize the __version__ = major.minor.micro.{revision} tag. 478 Overwrite C{sourcefile} in place by substituting the {revision} macro. 479 480 @param revision: revision number to write to the source file. 481 @type revision: int 482 @param sourcefile: python source file with a __version__ tag, typically 483 "csb/__init__.py" 484 @type sourcefile: str 485 486 @return: sourcefile.__version__ 487 """ 488 content = open(sourcefile).readlines() 489 490 with open(sourcefile, 'w') as src: 491 for line in content: 492 if line.startswith('__version__'): 493 src.write(line.format(revision=revision)) 494 else: 495 src.write(line) 496 497 self._delcache(sourcefile) 498 return imp.load_source('____source', sourcefile).__version__
499
500 - def _run(self, cmd):
501 502 si = Shell.run(cmd) 503 if si.code > 0: 504 raise RevisionError('SVN failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd) 505 506 return si.stdout.splitlines()
507
508 - def _delcache(self, sourcefile):
509 510 compiled = os.path.splitext(sourcefile)[0] + '.pyc' 511 if os.path.isfile(compiled): 512 os.remove(compiled) 513 514 pycache = os.path.join(os.path.dirname(compiled), '__pycache__') 515 if os.path.isdir(pycache): 516 shutil.rmtree(pycache)
517
518 -class RevisionInfo(object):
519
520 - def __init__(self, item, revision, maxrevision):
521 522 self.item = item 523 self.revision = revision 524 self.maxrevision = maxrevision
525
526 527 -def main():
528 Console.run()
529 530 531 if __name__ == '__main__': 532 533 main() 534