1 """
2 CSB build related tools and programs.
3
4 When executed as a program, this module will run the CSB Build Console and
5 build the source tree it belongs to. The source tree is added at the
6 B{beginning} of sys.path to make sure that all subsequent imports from the
7 Test and Doc consoles will import the right thing (think of multiple CSB
8 packages installed on the same server).
9
10 Here is how to build, test and package the whole project::
11
12 $ svn checkout https://svn.tuebingen.mpg.de/agbs/projects/CSB
13 $ CSB/trunk/csb/build.py -o <output directory>
14
15 The Console can also be imported and instantiated as a regular Python class.
16 In this case the Console again builds the source tree it is part of, but
17 sys.path will remain intact. Therefore, the Console will assume that all
18 modules currently in memory, as well as those that can be subsequently imported
19 by the Console itself, belong to the same CSB package.
20
21 @note: The CSB build services no longer support the option to build external
22 source trees.
23 @see: [CSB 0000038]
24 """
25 from __future__ import print_function
26
27 import os
28 import sys
29 import getopt
30 import traceback
31
32 if os.path.basename(__file__) == '__init__.py':
33 PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
34 else:
35 PARENT = os.path.abspath(os.path.dirname(__file__))
36
37 ROOT = 'csb'
38 SOURCETREE = os.path.abspath(os.path.join(PARENT, ".."))
39
40 if __name__ == '__main__':
41
42
43
44 for path in sys.path:
45 if path.startswith(SOURCETREE):
46 sys.path.remove(path)
47
48 import io
49 assert hasattr(io, 'BufferedIOBase')
50
51 sys.path = [SOURCETREE] + sys.path
52
53
54 """
55 It is now safe to import any modules
56 """
57 import imp
58 import shutil
59 import tarfile
60
61 import csb
62
63 from csb.io import Shell
67 """
68 Enumeration of build types.
69 """
70
71 SOURCE = 'source'
72 BINARY = 'binary'
73
74 _du = { SOURCE: 'sdist', BINARY: 'bdist' }
75
76 @staticmethod
78 try:
79 return BuildTypes._du[key]
80 except KeyError:
81 raise ValueError('Unhandled build type: {0}'.format(key))
82
85 """
86 CSB Build Bot. Run with -h for usage.
87
88 @param output: build output directory
89 @type output: str
90 @param verbosity: verbosity level
91 @type verbosity: int
92
93 @note: The build console automatically detects and builds the csb package
94 it belongs to. You cannot build a different source tree with it.
95 See the module documentation for more info.
96 """
97
98 PROGRAM = __file__
99
100 USAGE = r"""
101 CSB Build Console: build, test and package the entire csb project.
102
103 Usage:
104 python {program} -o output [-v verbosity] [-t type] [-h]
105
106 Options:
107 -o output Build output directory
108 -v verbosity Verbosity level, default is 1
109 -t type Build type:
110 source - build source code distribution (default)
111 binary - build executable
112 -h, --help Display this help
113 """
114
116
117 self._input = None
118 self._output = None
119 self._temp = None
120 self._docs = None
121 self._apidocs = None
122 self._root = None
123 self._verbosity = None
124 self._type = buildtype
125 self._dist = BuildTypes.get(buildtype)
126
127 if os.path.join(SOURCETREE, ROOT) != PARENT:
128 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT))
129 self._input = SOURCETREE
130
131 self.output = output
132 self.verbosity = verbosity
133
134 @property
137
138 @property
141 @output.setter
143
144 self._output = os.path.abspath(value)
145 self._temp = os.path.join(self._output, 'build')
146 self._docs = os.path.join(self._temp, 'docs')
147 self._apidocs = os.path.join(self._docs, 'api')
148 self._root = os.path.join(self._temp, ROOT)
149
150 @property
152 return self._verbosity
153 @verbosity.setter
155 self._verbosity = int(value)
156
158 """
159 Run the console.
160 """
161 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE))
162
163 self._init()
164 v = self._revision()
165 self._doc(v)
166 self._test()
167 vn = self._package()
168
169 self.log('\n# Done ({0}).\n'.format(vn.full))
170
171 - def log(self, message, level=1, ending='\n'):
172
173 if self._verbosity >= level:
174 sys.stdout.write(message)
175 sys.stdout.write(ending)
176 sys.stdout.flush()
177
179 """
180 Collect all required stuff in the output folder.
181 """
182 self.log('# Preparing the file system...')
183
184 if not os.path.exists(self._output):
185 self.log('Creating output directory {0}'.format(self._output), level=2)
186 os.mkdir(self._output)
187
188 if os.path.exists(self._temp):
189 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)
190 shutil.rmtree(self._temp)
191
192 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)
193 shutil.copytree(self._input, self._temp)
194
195 if os.path.exists(self._apidocs):
196 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)
197 shutil.rmtree(self._apidocs)
198 if not os.path.isdir(self._docs):
199 self.log('Creating docs directory {0}'.format(self._docs), level=2)
200 os.mkdir(self._docs)
201 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)
202 os.mkdir(self._apidocs)
203
205 """
206 Write the actual revision number to L{ROOT}.__version__
207 """
208 self.log('\n# Setting the most recent Revision Number...')
209 root = os.path.join(self._root, '__init__.py')
210
211 self.log('Retrieving revision number from {0}'.format(root), level=2)
212 rh = RevisionHandler(root)
213 revision = rh.read().maxrevision
214
215 self.log('Writing back revision number {0}'.format(revision), level=2)
216 version = rh.write(revision, root)
217
218 self.log(' This is {0}.__version__ {1}'.format(ROOT, version), level=1)
219 csb.__version__ = version
220
221 return version
222
250
251 - def _doc(self, version):
252 """
253 Build documentation in the output folder.
254 """
255 self.log('\n# Generating API documentation...')
256 try:
257 import epydoc.cli
258 except ImportError:
259 self.log('\n Skipped: epydoc is missing')
260 return
261
262 self.log('\n# Emulating ARGV for the Doc Builder...', level=2)
263 argv = sys.argv
264 sys.argv = ['epydoc', '--html', '-o', self._apidocs,
265 '--name', '{0} v{1}'.format(ROOT.upper(), version),
266 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases',
267 '--css', os.path.join(self._temp, 'epydoc.css'),
268 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning',
269 self._root]
270
271 if self._verbosity > 0:
272 sys.argv.append('-v')
273
274 try:
275 epydoc.cli.cli()
276 sys.exit(0)
277 except SystemExit as ex:
278 if ex.code is 0:
279 self.log('\n Passed all doc tests')
280 else:
281 if ex.code == 2:
282 self.log('\n DID NOT PASS: The docs might be broken')
283 else:
284 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex))
285
286 self.log('\n# Restoring the previous ARGV...', level=2)
287 sys.argv = argv
288
290 """
291 Make package.
292 """
293 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2)
294 cwd = os.curdir
295 os.chdir(self._temp)
296
297 if self._verbosity > 1:
298 verbosity = '-v'
299 else:
300 verbosity = '-q'
301 argv = sys.argv
302 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]
303
304 self.log('\n# Building {0} distribution...'.format(self._type))
305 try:
306 setup = imp.load_source('setupcsb', 'setup.py')
307 d = setup.build()
308 version = setup.VERSION
309 package = d.dist_files[0][2]
310
311 if self._type == BuildTypes.BINARY:
312 self._strip_source(package)
313
314 except SystemExit as ex:
315 if ex.code is not 0:
316 package = 'FAIL'
317 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex))
318
319 self.log('\n# Restoring the previous CWD and ARGV...', level=2)
320 os.chdir(cwd)
321 sys.argv = argv
322
323 self.log(' Packaged ' + package)
324 return version
325
327 """
328 Delete plain text source code files from the package.
329 """
330 cwd = os.getcwd()
331
332 try:
333 tmp = os.path.join(self.output, 'tmp')
334 os.mkdir(tmp)
335
336 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)
337 os.chdir(tmp)
338
339 oldtar = tarfile.open(package, mode='r:gz')
340 oldtar.extractall(tmp)
341 oldtar.close()
342
343 newtar = tarfile.open(package, mode='w:gz')
344
345 try:
346 for i in os.walk('.'):
347 for fn in i[2]:
348 if fn.endswith('.py'):
349 module = os.path.join(i[0], fn);
350 if not os.path.isfile(module.replace('.py', '.pyc')):
351 raise ValueError('Missing bytecode for module {0}'.format(module))
352 else:
353 os.remove(os.path.join(i[0], fn))
354
355 for i in os.listdir('.'):
356 newtar.add(i)
357 finally:
358 newtar.close()
359
360 finally:
361 self.log('\n# Restoring the previous CWD...', level=2)
362 os.chdir(cwd)
363 if os.path.exists(tmp):
364 shutil.rmtree(tmp)
365
366 @staticmethod
367 - def exit(message=None, code=0, usage=True):
375
376 @staticmethod
377 - def run(argv=None):
378
379 if argv is None:
380 argv = sys.argv[1:]
381
382 output = None
383 verb = 1
384 buildtype = BuildTypes.SOURCE
385
386 try:
387 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help'])
388
389 for option, value in options:
390 if option in('-h', '--help'):
391 Console.exit(message=None, code=0)
392 if option in('-o', '--output'):
393 if not os.path.isdir(value):
394 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3)
395 output = value
396 if option in('-v', '--verbosity'):
397 try:
398 verb = int(value)
399 except ValueError:
400 Console.exit(message='E: Verbosity must be an integer.', code=4)
401 if option in('-t', '--type'):
402 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]:
403 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5)
404 buildtype = value
405 except getopt.GetoptError as oe:
406 Console.exit(message='E: ' + str(oe), code=1)
407
408 if not output:
409 Console.exit(code=1, usage=True)
410 else:
411 try:
412 Console(output, verbosity=verb, buildtype=buildtype).build()
413 except Exception as ex:
414 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc())
415 Console.exit(message=msg, code=99, usage=False)
416
425
427 """
428 Determines the current SVN revision number of a working copy.
429
430 @param path: a local checkout path to be examined
431 @type path: str
432 @param svn: name of the svn program
433 @type svn: str
434 """
435
437
438 self.path = None
439 self.svn = None
440
441 if os.path.exists(path):
442 self.path = path
443 else:
444 raise IOError('Path not found: {0}'.format(path))
445 if Shell.run([svn, 'help']).code is 0:
446 self.svn = svn
447 else:
448 raise RevisionError('SVN probe failed', None, None)
449
451 """
452 Return the current revision information.
453 @rtype: L{RevisionInfo}
454
455 @todo: we can easily extend the svn output parser to grab more attributes,
456 say URL, author, etc. RevisionInfo would also has to be extended
457 """
458 cmd = '{0.svn} info {0.path}'.format(self)
459 revision = 0
460 maxrevision = 0
461
462 for line in self._run(cmd):
463 if line.startswith('Revision:'):
464 revision = int(line[9:] .strip())
465 break
466
467 for line in self._run(cmd + ' -R'):
468 if line.startswith('Revision:'):
469 rev = int(line[9:] .strip())
470 if rev > maxrevision:
471 maxrevision = rev
472
473 return RevisionInfo(self.path, revision, maxrevision)
474
475 - def write(self, revision, sourcefile):
476 """
477 Finalize the __version__ = major.minor.micro.{revision} tag.
478 Overwrite C{sourcefile} in place by substituting the {revision} macro.
479
480 @param revision: revision number to write to the source file.
481 @type revision: int
482 @param sourcefile: python source file with a __version__ tag, typically
483 "csb/__init__.py"
484 @type sourcefile: str
485
486 @return: sourcefile.__version__
487 """
488 content = open(sourcefile).readlines()
489
490 with open(sourcefile, 'w') as src:
491 for line in content:
492 if line.startswith('__version__'):
493 src.write(line.format(revision=revision))
494 else:
495 src.write(line)
496
497 self._delcache(sourcefile)
498 return imp.load_source('____source', sourcefile).__version__
499
500 - def _run(self, cmd):
501
502 si = Shell.run(cmd)
503 if si.code > 0:
504 raise RevisionError('SVN failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd)
505
506 return si.stdout.splitlines()
507
509
510 compiled = os.path.splitext(sourcefile)[0] + '.pyc'
511 if os.path.isfile(compiled):
512 os.remove(compiled)
513
514 pycache = os.path.join(os.path.dirname(compiled), '__pycache__')
515 if os.path.isdir(pycache):
516 shutil.rmtree(pycache)
517
519
520 - def __init__(self, item, revision, maxrevision):
521
522 self.item = item
523 self.revision = revision
524 self.maxrevision = maxrevision
525
529
530
531 if __name__ == '__main__':
532
533 main()
534