Source code for utool.util_setup

# -*- coding: utf-8 -*-
# Utools for setup.py files
from __future__ import absolute_import, division, print_function
import sys
import textwrap
from os.path import exists, join, dirname, split, splitext
import os
from utool import util_cplat
from utool import util_path
from utool import util_io
from utool import util_str

# from utool import util_dev
from utool.util_dbg import printex
from utool._internal import meta_util_arg

VERBOSE = meta_util_arg.VERBOSE


[docs]class SetupManager(object): """ Helps with writing setup.py """ def __init__(self): self.cmdclass = {} def _register_command(self, name, func): import setuptools class _WrapCommand(setuptools.Command): """ https://dankeder.com/posts/adding-custom-commands-to-setup-py/ """ description = name user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(cmd): func() self.cmdclass[name] = _WrapCommand # self.cmdclass[name] = func # TmpCommand
[docs] def register_command(self, name): import utool as ut if ut.is_funclike(name): func = name name = ut.get_funcname(func) self._register_command(name, func) return func else: def _wrap(func): self._register_command(name, func) return _wrap
[docs] def get_cmdclass(self): cmdclass = self.cmdclass.copy() cmdclass.update(**get_cmdclass()) return cmdclass
[docs]class SETUP_PATTERNS: clutter_pybuild = [ '*.pyc', '*.pyo', ] clutter_cyth = [ '_*_cyth.o', '_*_cyth_bench.py', 'run_cyth_benchmarks.sh', '_*_cyth.c', '_*_cyth.pyd', '_*_cyth.pxd', '_*_cyth.html', '_*_cyth.pyx', '_*_cyth.so', '_*_cyth.dylib', ] chmod_test = ['test_*.py']
[docs]def read_license(license_file): try: with open(license_file, 'r') as file_: firstline = file_.readline() license_type = firstline.replace('License', '').strip() return license_type except IOError: print('Warning no LICENCE file') return '???'
[docs]def build_pyo(project_dirs): pyexe = util_cplat.python_executable() for dir_ in project_dirs: # command_args = [pyexe, '-O', '-m', 'compileall', dir_ + '\*.py'] command_args = [pyexe, '-O', '-m', 'compileall', dir_] # command = ' '.join(command_args) # os.system(command) util_cplat.shell(command_args)
[docs]def setup_chmod(setup_fpath, setup_dir, chmod_patterns): """Gives files matching pattern the same chmod flags as setup.py""" # st_mode = os.stat(setup_fpath).st_mode st_mode = 33277 for pattern in chmod_patterns: for fpath in util_path.glob(setup_dir, pattern, recursive=True): print('[setup] chmod fpath=%r' % fpath) os.chmod(fpath, st_mode)
[docs]def assert_in_setup_repo(setup_fpath, name=''): """pass in __file__ from setup.py""" setup_dir, setup_fname = split(setup_fpath) cwd = os.getcwd() # repo_dname = split(setup_dir)[1] # print('cwd = %r' % (cwd)) # print('repo_dname = %r' % repo_dname) # print('setup_dir = %r' % (setup_dir)) # print('setup_fname = %r' % (setup_fname)) try: assert setup_fname == 'setup.py', 'name is not setup.py' # assert name == '' or repo_dname == name, ('name=%r' % name) assert cwd == setup_dir, 'cwd is not setup_dir' assert exists(setup_dir), 'setup dir does not exist' assert exists(join(setup_dir, 'setup.py')), 'setup.py does not exist' except AssertionError as ex: printex(ex, 'ERROR!: setup.py must be run from repository root') raise
[docs]def clean(setup_dir, clutter_patterns, clutter_dirs): print('[setup] clean()') clutter_patterns_ = [pat for pat in clutter_patterns if not pat.endswith('/')] _clutter_dirs = [pat[:-1] for pat in clutter_patterns if pat.endswith('/')] clutter_dirs_ = _clutter_dirs + clutter_dirs util_path.remove_files_in_dir( setup_dir, clutter_patterns_, recursive=True, verbose=VERBOSE ) for dir_ in clutter_dirs_: util_path.delete(dir_, verbose=VERBOSE, print_exists=False)
# util_path.remove_files_in_dir(dir_) # for fpath in cython_files: # fname, ext = splitext(fpath) # for libext in util_cplat.LIB_EXT_LIST: # util_path.remove_file(fname + libext) # util_path.remove_file(fname + '.c') # def build_cython(cython_files): # """ doesn't work """ # for fpath in cython_files: # util_dev.compile_cython(fpath) # def translate_cyth(): # import cyth # cyth.translate_all()
[docs]def get_numpy_include_dir(): try: import numpy as np return np.get_include() except ImportError: return ''
[docs]def find_ext_modules(disable_warnings=True): from setuptools import Extension import utool from os.path import relpath cwd = os.getcwd() # CYTH = 'cyth' in sys.argv BEXT = 'bext' in sys.argv BUILD = 'build' in sys.argv BUILD_EXT = 'build_ext' in sys.argv # if any([BEXT, CYTH]): # translate_cyth() # translate cyth before finding ext modules if not any([BEXT, BUILD, BUILD_EXT]): # dont find modules if they are not being built return [] # pyx_list = utool.glob(cwd, '*_cython.pyx', recursive=True) pyx_list = utool.glob(cwd, '*.pyx', recursive=True) if disable_warnings: extra_compile_args = ['-Wno-format', '-Wno-unused-function'] else: extra_compile_args = [] ext_modules = [] for pyx_abspath in pyx_list: pyx_relpath = relpath(pyx_abspath, cwd) pyx_modname, _ = splitext(pyx_relpath.replace('\\', '.').replace('/', '.')) print('[find_ext] Found Module:') print(' * pyx_modname = %r' % (pyx_modname,)) print(' * pyx_relpath = %r' % (pyx_relpath,)) extmod = Extension( pyx_modname, [pyx_relpath], include_dirs=[get_numpy_include_dir()], extra_compile_args=extra_compile_args, ) ext_modules.append(extmod) return ext_modules
[docs]def find_packages(recursive=True, maxdepth=None): """ Finds all directories with an __init__.py file in them """ import utool if utool.VERBOSE: print( '[util_setup] find_packages(recursive=%r, maxdepth=%r)' % (recursive, maxdepth) ) from os.path import relpath cwd = os.getcwd() init_files = utool.glob(cwd, '__init__.py', recursive=recursive, maxdepth=maxdepth) package_paths = list(map(dirname, init_files)) package_relpaths = [relpath(path, cwd) for path in package_paths] packages = [] for path in package_relpaths: base = utool.dirsplit(path)[0] if exists(join(base, '__init__.py')): package = path.replace('/', '.').replace('\\', '.') packages.append(package) return packages
[docs]def get_cmdclass(): """DEPRICATE""" try: from Cython.Distutils import build_ext cmdclass = {'build_ext': build_ext} return cmdclass except Exception as ex: print(ex) print( 'WARNING: Cython is not installed. This is only a problem if you are building C extensions' ) return {}
[docs]def parse_author(): """TODO: this function should parse setup.py or a module for the author variable """ return 'Jon Crall' # FIXME
[docs]def autogen_sphinx_apidoc(): r""" autogen_sphinx_docs.py Ignore: C:\Python27\Scripts\autogen_sphinx_docs.py autogen_sphinx_docs.py pip uninstall sphinx pip install sphinx pip install sphinxcontrib-napoleon pip install sphinx --upgrade pip install sphinxcontrib-napoleon --upgrade cd C:\Python27\Scripts ls C:\Python27\Scripts python -c "import sphinx; print(sphinx.__version__)" CommandLine: python -m utool.util_setup --exec-autogen_sphinx_apidoc Example: >>> # DISABLE_DOCTEST >>> # SCRIPT >>> from utool.util_setup import * # NOQA >>> autogen_sphinx_apidoc() """ # TODO: assert sphinx-apidoc exe is found # TODO: make find_exe work? import utool as ut def build_sphinx_apidoc_cmdstr(): print('') print('if this fails try: sudo pip install sphinx') print('') apidoc = 'sphinx-apidoc' if ut.WIN32: winprefix = 'C:/Python27/Scripts/' sphinx_apidoc_exe = winprefix + apidoc + '.exe' else: sphinx_apidoc_exe = apidoc apidoc_argfmt_list = [ sphinx_apidoc_exe, '--force', '--full', '--maxdepth="{maxdepth}"', '--doc-author="{author}"', '--doc-version="{doc_version}"', '--doc-release="{doc_release}"', '--output-dir="_doc"', #'--separate', # Put documentation for each module on its own page '--private', # Include "_private" modules '{pkgdir}', ] outputdir = '_doc' author = ut.parse_author() packages = ut.find_packages(maxdepth=1) assert len(packages) != 0, 'directory must contain at least one package' if len(packages) > 1: assert ( len(packages) == 1 ), 'FIXME I dont know what to do with more than one root package: %r' % ( packages, ) pkgdir = packages[0] version = ut.parse_package_for_version(pkgdir) modpath = dirname(ut.truepath(pkgdir)) apidoc_fmtdict = { 'author': author, 'maxdepth': '8', 'pkgdir': pkgdir, 'doc_version': version, 'doc_release': version, 'outputdir': outputdir, } ut.assert_exists('setup.py') ut.ensuredir('_doc') apidoc_fmtstr = ' '.join(apidoc_argfmt_list) apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict) print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir,)) if ut.VERBOSE: print(ut.repr4(apidoc_fmtdict)) return apidoc_cmdstr, modpath, outputdir def build_conf_replstr(): # # Make custom edits to conf.py # FIXME: # ext_search_text = ut.unindent( # r''' # extensions = [ # [^\]]* # ] # ''') ext_search_text = r'extensions = \[[^/]*\]' # TODO: http://sphinx-doc.org/ext/math.html#module-sphinx.ext.pngmath #'sphinx.ext.mathjax', exclude_modules = [] ext_repl_text = ut.codeblock( """ MOCK_MODULES = {exclude_modules} if len(MOCK_MODULES) > 0: import mock for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', # For LaTeX 'sphinx.ext.pngmath', # For Google Sytle Docstrs # https://pypi.python.org/pypi/sphinxcontrib-napoleon 'sphinxcontrib.napoleon', #'sphinx.ext.napoleon', ] """ ).format(exclude_modules=str(exclude_modules)) # theme_search = 'html_theme = \'default\'' theme_search = "html_theme = '[a-zA-Z_1-3]*'" theme_repl = ut.codeblock( """ import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] """ ) head_text = ut.codeblock( """ from sphinx.ext.autodoc import between import sphinx_rtd_theme import sys import os # Dont parse IBEIS args os.environ['IBIES_PARSE_ARGS'] = 'OFF' os.environ['UTOOL_AUTOGEN_SPHINX_RUNNING'] = 'ON' sys.path.append('{modpath}') sys.path.append(sys.path.insert(0, os.path.abspath("../"))) autosummary_generate = True modindex_common_prefix = ['_'] """ ).format(modpath=ut.truepath(modpath)) tail_text = ut.codeblock( """ def setup(app): # Register a sphinx.ext.autodoc.between listener to ignore everything # between lines that contain the word IGNORE app.connect('autodoc-process-docstring', between('^.*IGNORE.*$', exclude=True)) return app """ ) return ( ext_search_text, ext_repl_text, theme_search, theme_repl, head_text, tail_text, ) apidoc_cmdstr, modpath, outputdir = build_sphinx_apidoc_cmdstr() ( ext_search_text, ext_repl_text, theme_search, theme_repl, head_text, tail_text, ) = build_conf_replstr() dry = ut.get_argflag('--dry') if not dry: # Execute sphinx-apidoc ut.cmd(apidoc_cmdstr, shell=True) # sphinx-apidoc outputs conf.py to <outputdir>, add custom commands # # Change dir to <outputdir> print('chdir' + outputdir) os.chdir(outputdir) conf_fname = 'conf.py' conf_text = ut.read_from(conf_fname) conf_text = conf_text.replace('import sys', 'import sys # NOQA') conf_text = conf_text.replace('import os', 'import os # NOQA') conf_text = ut.regex_replace(theme_search, theme_repl, conf_text) conf_text = ut.regex_replace(ext_search_text, ext_repl_text, conf_text) conf_text = head_text + '\n' + conf_text + tail_text ut.write_to(conf_fname, conf_text) # Make the documentation # if ut.LINUX: # ut.cmd('make html', shell=True) # if ut.WIN32: # raw_input('waiting') if not ut.get_argflag('--nomake'): ut.cmd('make', 'html', shell=True) else: print(apidoc_cmdstr) print('cd ' + outputdir) print('manual edits of conf.py') print('make html')
[docs]def NOOP(): pass
[docs]def presetup_commands(setup_fpath, kwargs): if VERBOSE: print('[setup] presetup_commands()') name = kwargs.get('name', '') # Parse args project_dirs = kwargs.pop('project_dirs', None) chmod_patterns = kwargs.pop('chmod_patterns', []) clutter_dirs = kwargs.pop('clutter_dirs', None) clutter_patterns = kwargs.pop('clutter_patterns', []) build_command = kwargs.pop('build_command', NOOP) # Augment patterns with builtin patterns chmod_patterns += SETUP_PATTERNS.chmod_test if kwargs.pop('chmod_tests', True) else [] clutter_patterns += ( SETUP_PATTERNS.clutter_pybuild if kwargs.pop('clean_pybuild', True) else [] ) clutter_patterns += ( SETUP_PATTERNS.clutter_cyth if kwargs.pop('clean_pybuild', True) else [] ) setup_fpath = util_path.truepath(setup_fpath) # setup_dir = dirname(setup_fpath) build_dir = join(setup_dir, 'build') os.chdir(setup_dir) # change into setup directory assert_in_setup_repo(setup_fpath, name) # Augment clutter dirs if clutter_dirs is None: clutter_dirs = [] clutter_dirs += ['build', 'dist', name + '.egg-info', '__pycache__'] if project_dirs is None: project_dirs = util_path.ls_moduledirs(setup_dir) # Execute pre-setup commands based on argv # BEXT = 'bext' in sys.argv # BUILD_EXT = 'build_ext' in sys.argv # CYTH = 'cyth' in sys.argv # if CYTH: # translate_cyth() for arg in iter(sys.argv[:]): # print(arg) # Clean clutter files if arg in ['clean']: clean(setup_dir, clutter_patterns, clutter_dirs) # sys.exit(0) if arg in ['build'] or (not exists(build_dir) and 'clean' not in sys.argv): if VERBOSE: print('[setup] Executing build command') try: build_command() except Exception as ex: printex(ex, 'Error calling buildcommand from cwd=%r\n' % os.getcwd()) raise if arg in ['versions']: print('+ ---') print('Testing preqres package versions') install_requires = kwargs.get('install_requires', []) import pip print('Checking install_requires = [\n%s\n]' % '\n'.join(install_requires)) pip.main(['show'] + [depline.split(' ')[0] for depline in install_requires]) print('L ___ Done Version Testing') if arg in ['docs']: # FIXME autogen_sphinx_apidoc() # Build optimized files if arg in ['o', 'pyo']: build_pyo(project_dirs) # Cythonize files # if arg in ['cython']: # build_cython(cython_files) # Chmod files if arg in ['chmod']: setup_chmod(setup_fpath, setup_dir, chmod_patterns) try: sys.argv.remove('docs') sys.argv.remove('cyth') # sys.argv.remove('--cyth-write') except ValueError: pass try: # SUPER HACK # aliases bext to build_ext --inplace sys.argv.remove('bext') sys.argv.append('build_ext') sys.argv.append('--inplace') except ValueError: pass
presetup = presetup_commands # def parse_package_version(fpath='__init__.py', varname='__version__'): # """ Statically parse the version number from __init__.py """ # # from os.path import dirname, join # import ast # # repo_dpath = dirname(__file__) # # file_fpath = join(repo_dpath, 'wbia', '__init__.py') # # file_fpath = join(repo_dpath, 'wbia', 'control', 'DB_SCHEMA_CURRENT.py') # with open(fpath) as file_: # sourcecode = file_.read() # pt = ast.parse(sourcecode) # class VersionVisitor(ast.NodeVisitor): # def visit_Assign(self, node): # for target in node.targets: # if target.id == varname: # self.version = node.value.s # visitor = VersionVisitor() # visitor.visit(pt) # return visitor.version
[docs]def parse_package_for_version(name): """ Searches for a variable named __version__ in name's __init__.py file and returns the value. This function parses the source text. It does not load the module. """ from utool import util_regex init_fpath = join(name, '__init__.py') version_errmsg = textwrap.dedent( """ You must include a __version__ variable in %s\'s __init__.py file. Try something like: __version__ = '1.0.0.dev1' """ % (name,) ) if not exists(init_fpath): raise AssertionError(version_errmsg) val_regex = util_regex.named_field('version', '[0-9a-zA-Z.]+') regexstr = "__version__ *= *['\"]" + val_regex def parse_version(line): # Helper line = line.replace(' ', '').replace('\t', '') match_dict = util_regex.regex_parse(regexstr, line) if match_dict is not None: return match_dict['version'] # Find the version in the text of the source # version = 'UNKNOWN_VERSION' with open(init_fpath, 'r') as file_: for line in file_.readlines(): if line.startswith('__version__'): version = parse_version(line) if version is not None: return version raise AssertionError(version_errmsg)
def __infer_setup_kwargs(module, kwargs): """Implicitly build kwargs based on standard info""" # Get project name from the module # if 'name' not in kwargs: # kwargs['name'] = module.__name__ # else: # raise AssertionError('must specify module name!') name = kwargs['name'] # Our projects depend on utool # if kwargs['name'] != 'utool': # install_requires = kwargs.get('install_requires', []) # if 'utool' not in install_requires: # install_requires.append('utool') # kwargs['install_requires'] = install_requires packages = kwargs.get('packages', []) if name not in packages: packages.append(name) kwargs['packages'] = packages if 'version' not in kwargs: version = parse_package_for_version(name) kwargs['version'] = version # Parse version # if 'version' not in kwargs: # if module is None: # version_errmsg = 'You must include a version (preferably one that matches the __version__ variable in your modules init file' # raise AssertionError(version_errmsg) # else: # Parse license if 'license' not in kwargs: try: kwargs['license'] = read_license('LICENSE') except IOError: pass # Parse readme if 'long_description' not in kwargs: kwargs['long_description'] = parse_readme()
[docs]def parse_readme(readmefile='README.md'): # DEPRICATE return util_io.read_from(readmefile, verbose=False, strict=False)
[docs]def setuptools_setup(setup_fpath=None, module=None, **kwargs): # TODO: Learn this better # https://docs.python.org/3.1/distutils/apiref.html # https://pythonhosted.org/an_example_pypi_project/setuptools.html # https://docs.python.org/2/distutils/setupscript.html https://docs.python.org/2/distutils/setupscript.html # Useful documentation: http://bashelton.com/2009/04/setuptools-tutorial/#setup.py-package_dir from utool.util_inject import inject_colored_exceptions inject_colored_exceptions() # Fluffly, but nice if VERBOSE: print(util_str.repr4(kwargs)) __infer_setup_kwargs(module, kwargs) presetup_commands(setup_fpath, kwargs) if VERBOSE: print(util_str.repr4(kwargs)) return kwargs
if __name__ == '__main__': """ CommandLine: python -m utool.util_setup python -m utool.util_setup --allexamples python -m utool.util_setup --allexamples --noface --nosrc """ import multiprocessing multiprocessing.freeze_support() # for win32 import utool as ut # NOQA ut.doctest_funcs()