# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import ast
import functools
import inspect
import os
import os.path
import re
import six
import sys
import types
from os.path import dirname
from six.moves import builtins
from collections import OrderedDict
from six.moves import range, zip # NOQA
from utool import util_regex
from utool import util_arg
from utool import util_inject
from utool import util_class
from utool._internal import meta_util_six
print, rrr, profile = util_inject.inject2(__name__)
VERBOSE_INSPECT, VERYVERB_INSPECT = util_arg.get_module_verbosity_flags('inspect')
LIB_PATH = dirname(os.__file__)
# def check_dynamic_member_vars(self):
# return {name: attr for name, attr in self.__dict__.items()
# if not name.startswith("__") and not callable(attr) and not type(attr) is staticmethod}
[docs]@util_class.reloadable_class
class BaronWraper(object):
def __init__(self, sourcecode):
import redbaron
import utool as ut
with ut.Timer('building baron'):
self.baron = redbaron.RedBaron(sourcecode)
[docs] def print_diff(self, fpath=None):
import utool as ut
fpath = getattr(self, 'fpath', fpath)
assert fpath is not None, 'specify original file'
old_text = ut.readfrom(fpath)
new_text = self.to_string()
diff_text = ut.difftext(old_text, new_text, 1)
colored_diff_text = ut.color_diff_text(diff_text)
print(colored_diff_text)
[docs] def write(self, fpath=None):
import utool as ut
fpath = getattr(self, 'fpath', fpath)
assert fpath is not None, 'specify original file'
new_text = self.to_string()
ut.write_to(fpath, new_text)
[docs] @classmethod
def from_fpath(cls, fpath):
import utool as ut
sourcecode = ut.readfrom(fpath)
self = cls(sourcecode)
self.fpath = fpath
return self
[docs] def to_string(self):
text = self.baron.dumps()
return text
[docs] def defined_functions(self, recursive=True):
found = self.baron.find_all('def', recursive=recursive)
return found
# name_list = [node.name for node in found]
# return name_list
[docs] def find_usage(self, name):
found = self.baron.find_all('NameNode', value=name, recursive=True)
used_in = []
for node in found:
parent_func = self.find_root_function(node)
used_in.append(parent_func)
return used_in
[docs] def find_func(self, name):
return self.baron.find('def', name=name)
[docs] def find_root_function(self, node):
par = node.parent_find('def')
if par is None:
raise ValueError('no parent for node=%r' % (node.name))
elif par.indentation == '':
# Top level function
return par
elif par.parent is not None:
par2 = par.parent
if par2.type == 'class' and par2.indentation == '':
return par
else:
return self.find_root_function(par)
else:
raise ValueError('unknown error for node=%r' % (node.name))
[docs] def internal_call_graph(self, with_doctests=False):
""""""
import utool as ut
import networkx as nx
G = nx.DiGraph()
with ut.Timer('finding defed funcs'):
functions = self.defined_functions()
with ut.Timer('parsing docstrings'):
doc_nodes = {}
for func in functions:
if with_doctests and len(func) > 0:
if func[0].type == 'raw_string':
docstr_ = eval(func[0].value)
docstr = ut.unindent(docstr_)
docblocks = ut.parse_docblocks_from_docstr(docstr)
count = 0
for key, block in docblocks:
if key.startswith('Example'):
doctest = block
# docblocks['Example:']
docname = '<doctest%d>' % (count,) + func.name
doc_nodes[docname] = doctest
G.add_node(docname)
G.nodes[docname]['color'] = (1, 0, 0)
count += 1
with ut.Timer('building function call graph'):
func_names = [func.name for func in functions]
for callee in func_names:
# G.nodes[callee]['color'] = '0x000000'
G.add_node(six.text_type(callee))
import re
pat = re.compile(ut.regex_or(func_names))
found = self.baron.find_all('NameNode', value=pat, recursive=True)
for node in ut.ProgIter(
found, 'Searching for parent funcs', adjust=False, freq=1
):
parent_func = self.find_root_function(node)
caller = parent_func.name
callee = node.name
G.add_edge(six.text_type(callee), six.text_type(caller))
with ut.Timer('building doctest call graph'):
for func in ut.ProgIter(functions, lbl='doctest call graph'):
# Check for usage in doctests
for caller, doctest in doc_nodes.items():
if func.name in doctest:
# G.add_edge(caller, callee)
G.add_edge(callee, caller)
return G
[docs]def get_internal_call_graph(fpath, with_doctests=False):
"""
CommandLine:
python -m utool.util_inspect get_internal_call_graph --show --modpath=~/code/ibeis/ibeis/init/main_helpers.py --show
python -m utool.util_inspect get_internal_call_graph --show --modpath=~/code/dtool/dtool/depcache_table.py --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> fpath = ut.get_argval('--modpath', default='.')
>>> with_doctests = ut.get_argflag('--with_doctests')
>>> G = get_internal_call_graph(fpath, with_doctests)
>>> ut.quit_if_noshow()
>>> import wbia.plottool as pt
>>> pt.qt4ensure()
>>> pt.show_nx(G, fontsize=8, as_directed=False)
>>> z = pt.zoom_factory()
>>> p = pt.pan_factory()
>>> ut.show_if_requested()
"""
import utool as ut
fpath = ut.truepath(fpath)
sourcecode = ut.readfrom(fpath)
self = ut.BaronWraper(sourcecode)
G = self.internal_call_graph(with_doctests=with_doctests)
return G
[docs]def get_module_from_class(class_):
return sys.modules[class_.__module__]
[docs]def check_static_member_vars(class_, fpath=None, only_init=True):
r"""
class\_ can either be live object or a classname
# fpath = ut.truepath('~/code/ibeis/ibeis/viz/viz_graph2.py')
# classname = 'AnnotGraphWidget'
"""
# import ast
# import astor
import utool as ut
if isinstance(class_, six.string_types):
classname = class_
if fpath is None:
raise Exception('must specify fpath')
else:
# We were given a live object
if not isinstance(class_, type):
# We were given the class instance not the class
class_instance = class_
class_ = class_instance.__class__
classname = class_.__name__
if fpath is None:
module = ut.get_module_from_class(class_)
fpath = ut.get_modpath(module)
sourcecode = ut.readfrom(fpath)
import redbaron
# Pares a FULL syntax tree that keeps blockcomments
baron = redbaron.RedBaron(sourcecode)
for node in baron:
if node.type == 'class' and node.name == classname:
classnode = node
break
def find_parent_method(node):
par = node.parent_find('def')
if par is not None and par.parent is not None:
if par.parent.type == 'class':
return par
else:
return find_parent_method(par)
# TODO: Find inherited attrs
# classnode.inherit_from
# inhertied_attrs = ['parent']
# inhertied_attrs = []
class_methods = []
for node in classnode:
if node.type == 'def':
if only_init:
if node.name == '__init__':
class_methods.append(node)
else:
class_methods.append(node)
class_vars = []
self_vars = []
for method_node in class_methods:
self_var = method_node.arguments[0].dumps()
self_vars.append(self_var)
for assign in method_node.find_all('assignment'):
# method_node = find_parent_method(assign)
if assign.target.dumps().startswith(self_var + '.'):
class_vars.append(assign.target.value[1].dumps())
static_attrs = ut.unique(class_vars)
return static_attrs
# class_members = ut.unique(class_vars + class_methods + inhertied_attrs)
if False:
self_var = self_vars[0]
# Find everything that is used
complex_cases = []
simple_cases = []
all_self_ref = classnode.find_all(
'name_', value=re.compile('.*' + self_var + '\\.*')
)
for x in all_self_ref:
if x.parent.type == 'def_argument':
continue
if x.parent.type == 'atomtrailers':
atom = x.parent
if ut.depth(atom.fst()) <= 3:
simple_cases.append(atom)
else:
complex_cases.append(atom)
# print(ut.depth(atom.value.data))
# print(atom.value)
# print(atom.dumps())
# if len(atom.dumps()) > 200:
# break
accessed_attrs = []
for x in simple_cases:
if x.value[0].dumps() == self_var:
attr = x.value[1].dumps()
accessed_attrs.append(attr)
accessed_attrs = ut.unique(accessed_attrs)
ut.setdiff(accessed_attrs, class_vars)
# print('Missing Attrs: ' + str(ut.setdiff(accessed_attrs, class_members)))
# fst = baron.fst()
# node = (baron.node_list[54]) # NOQA
# [n.type for n in baron.node_list]
# generator = astor.codegen.SourceGenerator(' ' * 4)
# generator.visit(pt)
# resturctured_source = (''.join(generator.result))
# print(resturctured_source)
# visitor = ast.NodeVisitor()
# visitor.visit(pt)
# class SpecialVisitor(ast.NodeVisitor):
[docs]def get_funcnames_from_modpath(modpath, include_methods=True):
"""
Get all functions defined in module
"""
import utool as ut
if True:
import jedi
source = ut.read_from(modpath)
# script = jedi.Script(source=source, source_path=modpath, line=source.count('\n') + 1)
definition_list = jedi.names(source)
funcname_list = [
definition.name
for definition in definition_list
if definition.type == 'function'
]
if include_methods:
classdef_list = [
definition for definition in definition_list if definition.type == 'class'
]
defined_methods = ut.flatten(
[definition.defined_names() for definition in classdef_list]
)
funcname_list += [
method.name
for method in defined_methods
if method.type == 'function' and not method.name.startswith('_')
]
else:
import redbaron
# Pares a FULL syntax tree that keeps blockcomments
sourcecode = ut.read_from(modpath)
baron = redbaron.RedBaron(sourcecode)
funcname_list = [
node.name
for node in baron.find_all('def', recursive=include_methods)
if not node.name.startswith('_')
]
return funcname_list
# @profile
[docs]def check_module_usage(modpath_patterns):
"""
FIXME: not fully implmented
Desired behavior is ---
Given a set of modules specified by a list of patterns, returns how the
functions defined in the modules are called: a) with themselves and b) by
other files in the project not in the given set.
Args:
modpath_patterns (list):
CommandLine:
python -m utool.util_inspect check_module_usage --show
utprof.py -m utool.util_inspect check_module_usage --show
python -m utool.util_inspect check_module_usage --pat="['auto*', 'user_dialogs.py', 'special_query.py', 'qt_inc_automatch.py', 'devcases.py']"
python -m utool.util_inspect check_module_usage --pat="preproc_detectimg.py"
python -m utool.util_inspect check_module_usage --pat="neighbor_index.py"
python -m utool.util_inspect check_module_usage --pat="manual_chip_funcs.py"
python -m utool.util_inspect check_module_usage --pat="preproc_probchip.py"
python -m utool.util_inspect check_module_usage --pat="guiback.py"
python -m utool.util_inspect check_module_usage --pat="util_str.py"
Ignore:
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> modpath_patterns = ['_grave*']
>>> modpath_patterns = ['auto*', 'user_dialogs.py', 'special_query.py', 'qt_inc_automatch.py', 'devcases.py']
>>> modpath_patterns = ['neighbor_index.py']
>>> modpath_patterns = ['manual_chip_funcs.py']
>>> modpath_patterns = ut.get_argval('--pat', type_=list, default=['*'])
>>> result = check_module_usage(modpath_patterns)
>>> print(result)
"""
import utool as ut
# dpath = '~/code/ibeis/ibeis/algo/hots'
modpaths = ut.flatten([ut.glob_projects(pat) for pat in modpath_patterns])
modpaths = ut.unique(modpaths)
modnames = ut.lmap(ut.get_modname_from_modpath, modpaths)
print('Checking usage of modules: ' + ut.repr3(modpaths))
# Mark as True is module is always explicitly imported
restrict_to_importing_modpaths = False
cache = {}
def find_where_module_is_imported(modname):
"""finds where a module was explicitly imported. (in most scenareos)"""
# Find places where the module was imported
patterns = ut.possible_import_patterns(modname)
# do modname grep with all possible import patterns
grepres = ut.grep_projects(patterns, new=True, verbose=False, cache=cache)
return grepres.found_fpath_list
def find_function_callers(funcname, importing_modpaths):
"""searches for places where a function is used"""
pattern = ('\\b' + funcname + '\\b',)
# Search which module uses each public member
grepres = ut.grep_projects(
pattern, new=True, verbose=False, cache=cache, fpath_list=importing_modpaths
)
# Exclude places where function is defined or call is commented out
nohit_patterns = [
r'^\s*def',
r'^\s*#',
r'\-\-exec\-',
r'\-\-test-',
r'^\s*python -m ',
r'^\s*python -m wbia ',
r'^\s*wbia ',
r'\-\-test\-[a-zA-z]*\.',
r'\-\-exec\-[a-zA-z]*\.',
]
nohit_patterns += [
r'^\s*\>\>\>',
]
filter_pat = ut.regex_or(nohit_patterns)
# import copy
# grepres_ = copy.deepcopy(grepres)
grepres.inplace_filter_results(filter_pat)
grepres.found_modnames = ut.lmap(
ut.get_modname_from_modpath, grepres.found_fpath_list
)
parent_numlines = ut.lmap(len, grepres.found_lines_list)
numcall_graph_ = dict(zip(grepres.found_modnames, parent_numlines))
# Remove self references
# ut.delete_keys(numcall_graph_, modnames)
return numcall_graph_, grepres
print('Find modules that use this the query modules')
# Note: only works for explicit imports
importing_modpaths_list = [
find_where_module_is_imported(modname) for modname in modnames
]
print('Find members of the query modules')
funcnames_list = [get_funcnames_from_modpath(modpath) for modpath in modpaths]
print('Building call graph')
cache = {}
func_numcall_graph = ut.ddict(dict)
grep_results = ut.ddict(dict)
# Extract public members from each module
exclude_self = ut.get_argflag('--exclude-self')
_iter = list(zip(modnames, modpaths, importing_modpaths_list, funcnames_list))
_iter = ut.ProgIter(_iter, lbl='Searching query module', bs=False)
for modname, modpath, importing_modpaths, funcname_list in _iter:
if not restrict_to_importing_modpaths:
importing_modpaths = None
# Search for each function in modpath
for funcname in ut.ProgIter(funcname_list, lbl='Searching funcs in query module'):
numcall_graph_, grepres = find_function_callers(funcname, importing_modpaths)
grep_results[modname][funcname] = grepres
if exclude_self:
if modname in numcall_graph_:
del numcall_graph_[modname]
func_numcall_graph[modname][funcname] = numcall_graph_
# Sort by incidence cardinality
# func_numcall_graph = ut.odict([(key, ut.sort_dict(val, 'vals', len)) for key, val in func_numcall_graph.items()])
# Sort by weighted degree
func_numcall_graph = ut.odict(
[
(key, ut.sort_dict(val, 'vals', lambda x: sum(x.values())))
for key, val in func_numcall_graph.items()
]
)
# Print out grep results in order
print('PRINTING GREP RESULTS IN ORDER')
for modname, num_callgraph in func_numcall_graph.items():
print('\n============\n')
for funcname in num_callgraph.keys():
print('\n============\n')
with ut.Indenter('[%s]' % (funcname,)):
grepres = grep_results[modname][funcname]
print(grepres)
# print(func_numcall_graph[modname][funcname])
print('PRINTING NUMCALLGRAPH IN ORDER')
# Print out callgraph in order
print('func_numcall_graph = %s' % (ut.repr3(func_numcall_graph),))
# importance_dict = {}
# import copy
# func_call_graph2 = copy.deepcopy(func_numcall_graph)
# #ignore_modnames = []
# ignore_modnames = ['wbia.algo.hots.multi_index', 'wbia.algo.hots._neighbor_experiment']
# num_callers = ut.ddict(dict)
# for modname, modpath in list(zip(modnames, modpaths)):
# subdict = func_call_graph2[modname]
# for funcname in subdict.keys():
# numcall_graph_ = subdict[funcname]
# ut.delete_keys(numcall_graph_, modnames)
# ut.delete_keys(numcall_graph_, ignore_modnames)
# num_callers[modname][funcname] = sum(numcall_graph_.values())
# print(ut.repr4(num_callers[modname], sorted_=True, key_order_metric='val'))
# # Check external usage
# unused_external = []
# grep_results2 = copy.deepcopy(grep_results)
# for modname, grepres_subdict in grep_results2.items():
# for funcname, grepres_ in grepres_subdict.items():
# idxs = ut.find_list_indexes(grepres_.found_modnames, modnames)
# idxs += ut.find_list_indexes(grepres_.found_modnames, ignore_modnames)
# idxs = list(ut.filter_Nones(idxs))
# ut.delete_items_by_index(grepres_, idxs)
# ut.delete_items_by_index(grepres_.found_modnames, idxs)
# if len(grepres_) > 0:
# print(grepres_.make_resultstr())
# else:
# unused_external += [funcname]
# print('internal grep')
# # Check internal usage
# unused_internal = []
# grep_results2 = copy.deepcopy(grep_results)
# for modname, grepres_subdict in grep_results2.items():
# for funcname, grepres_ in grepres_subdict.items():
# idxs = ut.filter_Nones(ut.find_list_indexes(grepres_.found_modnames, [modname]))
# idxs_ = ut.index_complement(idxs, len(grepres_.found_modnames))
# ut.delete_items_by_index(grepres_, idxs_)
# ut.delete_items_by_index(grepres_.found_modnames, idxs_)
# grepres_.hack_remove_pystuff()
# #self = grepres_
# if len(grepres_) > 0:
# #print(modname)
# #print(funcname)
# #print(grepres_.extended_regex_list)
# print(grepres_.make_resultstr())
# else:
# unused_internal += [funcname]
# # HACK: how to write ut.parfor
# # returns a 0 lenth iterator so the for loop is never run. Then uses code
# # introspection to determine the content of the for loop body executes code
# # using the values of the local variables in a parallel / distributed
# # context.
# for modname, modpath in zip(modnames, modpaths):
# pattern = '\\b' + modname + '\\b',
# grepres = ut.grep_projects(pattern, new=True, verbose=False, cache=cache)
# parent_modnames = ut.lmap(ut.get_modname_from_modpath, grepres.found_fpath_list)
# parent_numlines = ut.lmap(len, grepres.found_lines_list)
# importance = dict(zip(parent_modnames, parent_numlines))
# ut.delete_keys(importance, modnames)
# importance_dict[modname] = importance
# print('importance_dict = %s' % (ut.repr3(importance_dict),))
# combo = reduce(ut.dict_union, importance_dict.values())
# print('combined %s' % (ut.repr3(combo),))
# print(ut.repr3(found_fpath_list))
pass
[docs]def get_object_methods(obj):
"""
Returns all methods belonging to an object instance specified in by the
__dir__ function
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> obj = ut.NiceRepr()
>>> methods1 = ut.get_object_methods()
>>> ut.inject_func_as_method(obj, ut.get_object_methods)
>>> methods2 = ut.get_object_methods()
>>> assert ut.get_object_methods in methods2
"""
import utool as ut
attr_list = (getattr(obj, attrname) for attrname in dir(obj))
methods = [attr for attr in attr_list if ut.is_method(attr)]
return methods
[docs]def help_members(obj, use_other=False):
r"""
Inspects members of a class
Args:
obj (class or module):
CommandLine:
python -m utool.util_inspect help_members
Ignore:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> obj = ut.DynStruct
>>> result = help_members(obj)
>>> print(result)
"""
import utool as ut
attrnames = dir(obj)
attr_list = [getattr(obj, attrname) for attrname in attrnames]
attr_types = ut.lmap(ut.type_str, map(type, attr_list))
unique_types, groupxs = ut.group_indices(attr_types)
type_to_items = ut.dzip(unique_types, ut.apply_grouping(attr_list, groupxs))
type_to_itemname = ut.dzip(unique_types, ut.apply_grouping(attrnames, groupxs))
# if memtypes is None:
# memtypes = list(type_to_items.keys())
memtypes = ['instancemethod'] # , 'method-wrapper']
func_mems = ut.dict_subset(type_to_items, memtypes, [])
func_list = ut.flatten(func_mems.values())
defsig_list = []
num_unbound_args_list = []
num_args_list = []
for func in func_list:
# args = ut.get_func_argspec(func).args
argspec = ut.get_func_argspec(func)
args = argspec.args
unbound_args = get_unbound_args(argspec)
defsig = ut.func_defsig(func)
defsig_list.append(defsig)
num_unbound_args_list.append(len(unbound_args))
num_args_list.append(len(args))
group = ut.hierarchical_group_items(
defsig_list, [num_unbound_args_list, num_args_list]
)
print(repr(obj))
print(ut.repr3(group, strvals=True))
if use_other:
other_mems = ut.delete_keys(type_to_items.copy(), memtypes)
other_mems_attrnames = ut.dict_subset(type_to_itemname, other_mems.keys())
named_other_attrs = ut.dict_union_combine(
other_mems_attrnames, other_mems, lambda x, y: list(zip(x, y))
)
print(ut.repr4(named_other_attrs, nl=2, strvals=True))
[docs]def get_dev_hints():
VAL_FIELD = util_regex.named_field('val', '.*')
VAL_BREF = util_regex.bref_field('val')
registered_hints = OrderedDict(
[
# General IBEIS hints
('ibs.*', ('wbia.IBEISController', 'image analysis api')),
('testres', ('wbia.TestResult', 'test result object')),
(
'qreq_',
('wbia.QueryRequest', 'query request object with hyper-parameters'),
),
('cm', ('wbia.ChipMatch', 'object of feature correspondences and scores')),
('qparams*', ('wbia.QueryParams', 'query hyper-parameters')),
('qaid2_cm.*', ('dict', 'dict of ``ChipMatch`` objects')),
('vecs', ('ndarray[uint8_t, ndim=2]', 'descriptor vectors')),
('maws', ('ndarray[float32_t, ndim=1]', 'multiple assignment weights')),
(
'words',
('ndarray[uint8_t, ndim=2]', 'aggregate descriptor cluster centers'),
),
('word', ('ndarray[uint8_t, ndim=1]', 'aggregate descriptor cluster center')),
('rvecs', ('ndarray[uint8_t, ndim=2]', 'residual vector')),
('fm', ('list', 'list of feature matches as tuples (qfx, dfx)')),
('fs', ('list', 'list of feature scores')),
('aid_list', ('list', 'list of annotation rowids')),
('aids', ('list', 'list of annotation rowids')),
('aid_list[0-9]', ('list', 'list of annotation ids')),
('ensure', ('bool', 'eager evaluation if True')),
('qaid', ('int', 'query annotation id')),
('aid[0-9]?', ('int', 'annotation id')),
('daids', ('list', 'database annotation ids')),
('qaids', ('list', 'query annotation ids')),
('use_cache', ('bool', 'turns on disk based caching')),
('qreq_vsmany_', ('QueryRequest', 'persistant vsmany query request')),
('qnid', ('int', 'query name id')),
#
('gfpath[0-9]?', ('str', 'image file path string')),
('path[0-9]?', ('str', 'path to file or directory')),
('n', ('int', '')),
('ext', ('str', 'extension')),
('_path', ('str', 'path string')),
('path_', ('str', 'path string')),
('.*_dpath', ('str', 'directory path string')),
('.*_fpath', ('str', 'file path string')),
('bbox', ('tuple', 'bounding box in the format (x, y, w, h)')),
('theta', ('float', 'angle in radians')),
('ori_thresh', ('float', 'angle in radians')),
('xy_thresh_sqrd', ('float', '')),
('xy_thresh', ('float', '')),
('scale_thresh', ('float', '')),
# Pipeline hints
('qaid2_nns', ('dict', 'maps query annotid to (qfx2_idx, qfx2_dist)')),
('qaid2_nnvalid0', ('dict', 'maps query annotid to qfx2_valid0')),
(
'qfx2_valid0',
(
'ndarray',
'maps query feature index to K matches non-impossibility flags',
),
),
('filt2_weights', ('dict', 'maps filter names to qfx2_weight ndarray')),
(
'qaid2_filtweights',
('dict', 'mapping to weights computed by filters like lnnbnn and ratio'),
),
(
'qaid2_nnfiltagg',
('dict', 'maps to nnfiltagg - tuple(qfx2_score, qfx2_valid)'),
),
(
'qaid2_nnfilts',
(
'dict',
'nonaggregate feature scores and validities for each feature NEW',
),
),
('nnfiltagg', ('tuple', '(qfx2_score_agg, qfx2_valid_agg)')),
('nnfilts', ('tuple', '(filt_list, qfx2_score_list, qfx2_valid_list)')),
(
'qfx2_idx',
(
'ndarray[int32_t, ndims=2]',
'mapping from query feature index to db neighbor index',
),
),
('K', ('int', None)),
('Knorm', ('int', None)),
# SMK Hints
('smk_alpha', ('float', 'selectivity power')),
('smk_thresh', ('float', 'selectivity threshold')),
('query_sccw', ('float', 'query self-consistency-criterion')),
('data_sccw', ('float', 'data self-consistency-criterion')),
('invindex', ('InvertedIndex', 'object for fast vocab lookup')),
# Plotting hints
(
'[qd]?rchip[0-9]?',
('ndarray[uint8_t, ndim=2]', 'rotated annotation image data'),
),
('[qd]?chip[0-9]?', ('ndarray[uint8_t, ndim=2]', 'annotation image data')),
('kp', ('ndarray[float32_t, ndim=1]', 'a single keypoint')),
('[qd]?kpts[0-9]?', ('ndarray[float32_t, ndim=2]', 'keypoints')),
('[qd]?vecs[0-9]?', ('ndarray[uint8_t, ndim=2]', 'descriptor vectors')),
('H', ('ndarray[float64_t, ndim=2]', 'homography/perspective matrix')),
('invV_mats2x2', ('ndarray[float32_t, ndim=3]', 'keypoint shapes')),
(
'invVR_mats2x2',
('ndarray[float32_t, ndim=3]', 'keypoint shape and rotations'),
),
(
'invV_mats',
('ndarray[float32_t, ndim=3]', 'keypoint shapes (possibly translation)'),
),
(
'invVR_mats',
(
'ndarray[float32_t, ndim=3]',
'keypoint shape and rotations (possibly translation)',
),
),
# ('img\d*', ('ndarray[uint8_t, ndim=2]', 'image data')),
('img_in', ('ndarray[uint8_t, ndim=2]', 'image data')),
('arr', ('ndarray', '')),
('arr_', ('ndarray', '')),
('X', ('ndarray', 'data')),
('y', ('ndarray', 'labels')),
(
'imgBGR',
(
'ndarray[uint8_t, ndim=2]',
'image data in opencv format (blue, green, red)',
),
),
('pnum', ('tuple', 'plot number')),
('fnum', ('int', 'figure number')),
('title', ('str', '')),
('text', ('str', '')),
('text_', ('str', '')),
# Matching Hints
('ratio_thresh', ('float', None)),
# utool hints
('func', ('function', 'live python function')),
('funcname', ('str', 'function name')),
('modname', ('str', 'module name')),
('argname_list', ('str', 'list of argument names')),
('return_name', ('str', 'return variable name')),
('dict_', ('dict_', 'a dictionary')),
('examplecode', ('str', None)),
# Numpy Hints
('shape', ('tuple', 'array dimensions')),
('chipshape', ('tuple', 'height, width')),
('rng', ('RandomState', 'random number generator')),
# Opencv hings
('dsize', ('tuple', 'width, height')),
('chipsize', ('tuple', 'width, height')),
# Standard Python Hints for my coding style
('.*_fn', ('func', None)),
('str_', ('str', None)),
('num_.*', ('int', None)),
('.*_str', ('str', None)),
('.*_?list_?', ('list', None)),
('.*_?dict_?', ('dict', None)),
# ('dict_?\d?' , ('dict', None)),
('.*_tup', ('tuple', None)),
('.*_sublist', ('list', None)),
('fpath[0-9]?', ('str', 'file path string')),
('chip[A-Z]*', ('ndarray', 'cropped image')),
('verbose', ('bool', 'verbosity flag')),
# Other hints for my coding style
('wx2_', ('dict', None)),
(
'qfx2_' + VAL_FIELD,
('ndarray', 'mapping from query feature index to ' + VAL_BREF),
),
('.*x2_.*', ('ndarray', None)),
('.+[^3]2_.*', ('dict', None)),
('dpath', ('str', 'directory path')),
('dname', ('str', 'directory name')),
('fpath', ('str', 'file path')),
('fname', ('str', 'file name')),
('pattern', ('str', '')),
]
)
return registered_hints
[docs]def infer_arg_types_and_descriptions(argname_list, defaults):
"""
Args:
argname_list (list):
defaults (list):
Returns:
tuple : (argtype_list, argdesc_list)
CommandLine:
python -m utool.util_inspect --test-infer_arg_types_and_descriptions
Ignore:
python -c "import utool; print(utool.auto_docstr('wbia.algo.hots.pipeline', 'build_chipmatches'))"
Example:
>>> # ENABLE_DOCTEST
>>> import utool
>>> argname_list = ['ibs', 'qaid', 'fdKfds', 'qfx2_foo']
>>> defaults = None
>>> tup = utool.infer_arg_types_and_descriptions(argname_list, defaults)
>>> argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup
"""
# import utool as ut
from utool import util_dev
# hacks for IBEIS
if True or util_dev.is_developer():
registered_hints = get_dev_hints()
# key = regex pattern
# val = hint=tuple(type_, desc_)
if defaults is None:
defaults = []
default_types = [type(val).__name__.replace('NoneType', 'None') for val in defaults]
num_defaults = len(defaults)
num_nodefaults = len(argname_list) - num_defaults
argtype_list = ['?'] * (num_nodefaults) + default_types
# defaults aligned with argtype_list and argdesc_list
argdefault_list = [None] * num_nodefaults + list(defaults)
hasdefault_list = [False] * num_nodefaults + [True] * num_defaults
argdesc_list = ['' for _ in range(len(argname_list))]
# use hints to build better docstrs
for argx in range(len(argname_list)):
# if argtype_list[argx] == '?' or argtype_list[argx] == 'None':
argname = argname_list[argx]
if argname is None:
# print('warning argname is None')
continue
for regex, hint in six.iteritems(registered_hints):
matchobj = re.match(
'^' + regex + '$', argname, flags=re.MULTILINE | re.DOTALL
)
if matchobj is not None:
type_ = hint[0]
desc_ = hint[1]
if type_ is not None:
if argtype_list[argx] == '?' or argtype_list[argx] == 'None':
argtype_list[argx] = type_
if desc_ is not None:
desc_ = matchobj.expand(desc_)
argdesc_list[argx] = ' ' + desc_
break
# append defaults to descriptions
for argx in range(len(argdesc_list)):
if hasdefault_list[argx]:
if isinstance(argdefault_list[argx], types.ModuleType):
defaultrepr = argdefault_list[argx].__name__
else:
defaultrepr = repr(argdefault_list[argx])
# import utool as ut
# ut.embed()
argdesc_list[argx] += '(default = %s)' % (defaultrepr,)
return argtype_list, argdesc_list, argdefault_list, hasdefault_list
[docs]def get_module_owned_functions(module):
"""
Replace with iter_module_doctesable (but change that name to be something better)
returns functions actually owned by the module
module = vtool.distance
"""
import utool as ut
list_ = []
for key, val in ut.iter_module_doctestable(module):
belongs = False
if hasattr(val, '__module__'):
belongs = val.__module__ == module.__name__
elif hasattr(val, 'func_globals'):
belongs = val.func_globals['__name__'] == module.__name__
if belongs:
list_.append(val)
return list_
[docs]def zzz_profiled_is_no():
pass
[docs]@profile
def zzz_profiled_is_yes():
pass
[docs]def iter_module_doctestable(
module,
include_funcs=True,
include_classes=True,
include_methods=True,
include_builtin=True,
include_inherited=False,
debug_key=None,
):
r"""
Yeilds doctestable live object form a modules
TODO: change name to iter_module_members
Replace with iter_module_doctesable (but change that name to be something
better)
Args:
module (module): live python module
include_funcs (bool):
include_classes (bool):
include_methods (bool):
include_builtin (bool): (default = True)
include_inherited (bool): (default = False)
Yeilds:
tuple (str, callable): (funcname, func) doctestable
CommandLine:
python -m utool --tf iter_module_doctestable \
--modname=wbia.algo.hots.chip_match
--modname=wbia.control.IBEISControl
--modname=wbia.control.SQLDatabaseControl
--modname=wbia.control.manual_annot_funcs
--modname=wbia.control.manual_annot_funcs
--modname=wbia.expt.test_result
--modname=utool.util_progress --debug-key=build_msg_fmtstr_time2
--modname=utool.util_progress --debug-key=ProgressIter
Debug:
# fix profile with doctest
utprof.py -m utool --tf iter_module_doctestable --modname=utool.util_inspect --debugkey=zzz_profiled_is_yes
utprof.py -m utool --tf iter_module_doctestable --modname=wbia.algo.hots.chip_match --debugkey=to_json
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> modname = ut.get_argval('--modname', type_=str, default=None)
>>> kwargs = ut.argparse_funckw(iter_module_doctestable)
>>> module = ut.util_tests if modname is None else ut.import_modname(modname)
>>> debug_key = ut.get_argval('--debugkey', type_=str, default=None)
>>> kwargs['debug_key'] = debug_key
>>> kwargs['include_inherited'] = True
>>> doctestable_list = list(iter_module_doctestable(module, **kwargs))
>>> func_names = sorted(ut.take_column(doctestable_list, 0))
>>> print(ut.repr4(func_names))
"""
import ctypes
types.BuiltinFunctionType
valid_func_types = [
types.FunctionType,
types.BuiltinFunctionType,
classmethod,
staticmethod,
types.MethodType, # handles classmethod
# types.BuiltinMethodType,
]
if include_builtin:
valid_func_types += [types.BuiltinFunctionType]
if six.PY2:
valid_class_types = (
types.ClassType,
types.TypeType,
)
else:
valid_class_types = six.class_types
scalar_types = [dict, list, tuple, set, frozenset, bool, float, int] + list(
six.string_types
)
scalar_types += list(six.string_types)
other_types = [functools.partial, types.ModuleType, ctypes.CDLL]
if six.PY2:
other_types += [types.InstanceType]
invalid_types = tuple(scalar_types + other_types)
valid_func_types = tuple(valid_func_types)
# modpath = ut.get_modname_from_modpath(module.__file__)
for key, val in six.iteritems(module.__dict__):
# <DEBUG>
if debug_key is not None and key == debug_key:
print('DEBUG')
print('debug_key = %r' % (debug_key,))
exec('item = val')
# import utool as ut
# ut.embed()
# </DEBUG>
if hasattr(val, '__module__'):
# HACK: todo. figure out true parent module
if val.__module__ == 'numpy':
continue
if val is None:
pass
elif isinstance(val, valid_func_types):
if include_funcs:
if not include_inherited and not is_defined_by_module(val, module):
continue
yield key, val
elif isinstance(val, valid_class_types):
class_ = val
if not include_inherited and not is_defined_by_module(class_, module):
continue
if include_classes:
# Yield the class itself
yield key, val
if include_methods:
# Yield methods of the class
for subkey, subval in six.iteritems(class_.__dict__):
if isinstance(subval, property):
subval = subval.fget
# <DEBUG>
if debug_key is not None and subkey == debug_key:
import utool as ut
ut.embed()
# </DEBUG>
# Unbound methods are still typed as functions
if isinstance(subval, valid_func_types):
if not include_inherited and not is_defined_by_module(
subval, module, parent=val
):
continue
if isinstance(subval, (staticmethod)):
subval.__func__.__ut_parent_class__ = class_
if not isinstance(
subval, types.BuiltinFunctionType
) and not isinstance(subval, (classmethod, staticmethod)):
# HACK: __ut_parent_class__ lets util_test have
# more info on the func should return extra info
# instead
subval.__ut_parent_class__ = class_
yield subkey, subval
elif isinstance(val, invalid_types):
pass
else:
if util_arg.VERBOSE:
print(
'[util_inspect] WARNING module %r class %r:'
% (module, class_)
)
print(' * Unknown if testable val=%r' % (val))
print(' * Unknown if testable type(val)=%r' % type(val))
elif isinstance(val, invalid_types):
pass
else:
# import utool as ut
if util_arg.VERBOSE:
print('[util_inspect] WARNING in module %r:' % (module,))
print(' * Unknown if testable val=%r' % (val))
print(' * Unknown if testable type(val)=%r' % type(val))
[docs]def is_defined_by_module2(item, module):
belongs = False
if hasattr(item, '__module__'):
belongs = item.__module__ == module.__name__
elif hasattr(item, 'func_globals'):
belongs = item.func_globals['__name__'] == module.__name__
return belongs
[docs]def is_defined_by_module(item, module, parent=None):
"""
Check if item is directly defined by a module.
This check may be prone to errors.
"""
flag = False
if isinstance(item, types.ModuleType):
if not hasattr(item, '__file__'):
try:
# hack for cv2 and xfeatures2d
import utool as ut
name = ut.get_modname_from_modpath(module.__file__)
flag = name in str(item)
except Exception:
flag = False
else:
item_modpath = os.path.realpath(dirname(item.__file__))
mod_fpath = module.__file__.replace('.pyc', '.py')
if not mod_fpath.endswith('__init__.py'):
flag = False
else:
modpath = os.path.realpath(dirname(mod_fpath))
modpath = modpath.replace('.pyc', '.py')
flag = item_modpath.startswith(modpath)
elif hasattr(item, '_utinfo'):
# Capture case where there is a utool wrapper
orig_func = item._utinfo['orig_func']
flag = is_defined_by_module(orig_func, module, parent)
else:
if isinstance(item, staticmethod):
# static methods are a wrapper around a function
item = item.__func__
try:
func_globals = meta_util_six.get_funcglobals(item)
func_module_name = func_globals['__name__']
if func_module_name == 'line_profiler':
valid_names = dir(module)
if parent is not None:
valid_names += dir(parent)
if item.func_name in valid_names:
# hack to prevent small names
# if len(item.func_name) > 8:
if len(item.func_name) > 6:
flag = True
elif func_module_name == module.__name__:
flag = True
except AttributeError:
if hasattr(item, '__module__'):
flag = item.__module__ == module.__name__
return flag
[docs]def get_func_modname(func):
if hasattr(func, '_utinfo'):
# Capture case where there is a utool wrapper
orig_func = func._utinfo['orig_func']
return get_func_modname(orig_func)
# try:
func_globals = meta_util_six.get_funcglobals(func)
modname = func_globals['__name__']
return modname
# except AttributeError:
# pass
# pass
[docs]def is_bateries_included(item):
"""
Returns if a value is a python builtin function
Args:
item (object):
Returns:
bool: flag
References:
http://stackoverflow.com/questions/23149218/check-if-a-python-function-is-builtin
CommandLine:
python -m utool._internal.meta_util_six is_builtin
Example:
>>> # DISABLE_DOCTEST
>>> from utool._internal.meta_util_six import * # NOQA
>>> item = zip
>>> flag = is_bateries_included(item)
>>> result = ('flag = %s' % (str(flag),))
>>> print(result)
"""
flag = False
if hasattr(item, '__call__') and hasattr(item, '__module__'):
if item.__module__ is not None:
module = sys.modules[item.__module__]
if module == builtins:
flag = True
elif hasattr(module, '__file__'):
flag = LIB_PATH == dirname(module.__file__)
return flag
[docs]def list_class_funcnames(fname, blank_pats=[' #']):
"""
list_class_funcnames
Args:
fname (str): filepath
blank_pats (list): defaults to ' #'
Returns:
list: funcname_list
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> fname = 'util_class.py'
>>> blank_pats = [' #']
>>> funcname_list = list_class_funcnames(fname, blank_pats)
>>> print(funcname_list)
"""
with open(fname, 'r') as file_:
lines = file_.readlines()
funcname_list = []
# full_line_ = ''
for lx, line in enumerate(lines):
# full_line_ += line
if any([line.startswith(pat) for pat in blank_pats]):
funcname_list.append('')
if line.startswith(' def '):
def_x = line.find('def')
rparen_x = line.find('(')
funcname = line[(def_x + 3) : rparen_x]
# print(funcname)
funcname_list.append(funcname)
return funcname_list
[docs]def list_global_funcnames(fname, blank_pats=[' #']):
"""
list_global_funcnames
Args:
fname (str): filepath
blank_pats (list): defaults to ' #'
Returns:
list: funcname_list
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> fname = 'util_class.py'
>>> blank_pats = [' #']
>>> funcname_list = list_global_funcnames(fname, blank_pats)
>>> print(funcname_list)
"""
with open(fname, 'r') as file_:
lines = file_.readlines()
funcname_list = []
# full_line_ = ''
for lx, line in enumerate(lines):
# full_line_ += line
if any([line.startswith(pat) for pat in blank_pats]):
funcname_list.append('')
if line.startswith('def '):
def_x = line.find('def')
rparen_x = line.find('(')
funcname = line[(def_x + 3) : rparen_x]
# print(funcname)
funcname_list.append(funcname)
return funcname_list
[docs]def inherit_kwargs(inherit_func):
"""
TODO move to util_decor
inherit_func = inspect_pdfs
func = encoder.visualize.im_func
"""
import utool as ut
keys, is_arbitrary = ut.get_kwargs(inherit_func)
if is_arbitrary:
keys += ['**kwargs']
kwargs_append = '\n'.join(keys)
# from six.moves import builtins
# builtins.print(kwargs_block)
def _wrp(func):
if func.__doc__ is None:
func.__doc__ = ''
# TODO append to kwargs block if it exists
kwargs_block = 'Kwargs:\n' + ut.indent(kwargs_append)
func.__doc__ += kwargs_block
return func
return _wrp
[docs]def filter_valid_kwargs(func, dict_):
import utool as ut
keys, is_arbitrary = ut.get_kwargs(func)
if is_arbitrary:
valid_dict_ = dict_
else:
key_subset = ut.dict_keysubset(dict_, keys)
valid_dict_ = ut.dict_subset(dict_, key_subset)
return valid_dict_
[docs]def dummy_func(arg1, arg2, arg3=None, arg4=[1, 2, 3], arg5={}, **kwargs):
"""
test func for kwargs parseing
"""
foo = kwargs.get('foo', None)
bar = kwargs.pop('bar', 4)
foo2 = kwargs['foo2']
foobar = str(foo) + str(bar) + str(foo2)
return foobar
[docs]def get_kwdefaults2(func, parse_source=False):
return get_kwdefaults(func, parse_source=True)
[docs]def get_kwdefaults(func, parse_source=False):
r"""
Args:
func (func):
Returns:
dict:
CommandLine:
python -m utool.util_inspect get_kwdefaults
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> func = dummy_func
>>> parse_source = True
>>> kwdefaults = get_kwdefaults(func, parse_source)
>>> print('kwdefaults = %s' % (ut.repr4(kwdefaults),))
"""
# import utool as ut
# with ut.embed_on_exception_context:
argspec = inspect.getfullargspec(func)
kwdefaults = {}
if argspec.args is None or argspec.defaults is None:
pass
else:
args = argspec.args
defaults = argspec.defaults
# kwdefaults = OrderedDict(zip(argspec.args[::-1], argspec.defaults[::-1]))
kwpos = len(args) - len(defaults)
kwdefaults = OrderedDict(zip(args[kwpos:], defaults))
if parse_source and argspec.varkw:
# TODO parse for kwargs.get/pop
keyword_defaults = parse_func_kwarg_keys(func, with_vals=True)
for key, val in keyword_defaults:
assert key not in kwdefaults, 'parsing error'
kwdefaults[key] = val
return kwdefaults
[docs]def get_argnames(func):
argspec = inspect.getfullargspec(func)
argnames = argspec.args
return argnames
[docs]def get_funcname(func):
return meta_util_six.get_funcname(func)
[docs]def set_funcname(func, newname):
return meta_util_six.set_funcname(func, newname)
[docs]def get_method_func(func):
if six.PY2:
return func.im_func
else:
return func.__func__
[docs]def get_funcglobals(func):
return meta_util_six.get_funcglobals(func)
[docs]def get_funcdoc(func):
return meta_util_six.get_funcdoc(func)
[docs]def get_funcfpath(func):
return func.func_code.co_filename
[docs]def set_funcdoc(func, newdoc):
return meta_util_six.set_funcdoc(func, newdoc)
[docs]def get_docstr(func_or_class):
"""Get the docstring from a live object"""
import utool as ut
try:
docstr_ = func_or_class.func_doc
except AttributeError:
docstr_ = func_or_class.__doc__
if docstr_ is None:
docstr_ = ''
docstr = ut.unindent(docstr_)
return docstr
[docs]def get_func_docblocks(func_or_class):
import utool as ut
docstr = ut.get_docstr(func_or_class)
docblocks = ut.parse_docblocks_from_docstr(docstr)
return docblocks
[docs]def prettyprint_parsetree(pt):
"""
pip install astdump
pip install codegen
"""
# import astdump
import astor
# import codegen
# import ast
# astdump.indented(pt)
# print(ast.dump(pt, include_attributes=True))
print(astor.dump(pt))
[docs]def special_parse_process_python_code(sourcecode):
r"""
pip install redbaron
http://stackoverflow.com/questions/7456933/python-ast-with-preserved-comments
CommandLine:
python -m utool.util_inspect special_parse_process_python_code --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> sourcecode = ut.read_from(ut.util_inspect.__file__)
>>> result = special_parse_process_python_code(sourcecode)
>>> print(result)
"""
import ast
import astor
# sourcecode = 'from __future__ import print_function\n' + sourcecode
sourcecode_ = sourcecode.encode('utf8')
pt = ast.parse(sourcecode_, 'testfile')
generator = astor.codegen.SourceGenerator(' ' * 4)
generator.visit(pt)
resturctured_source = ''.join(generator.result)
print(resturctured_source)
visitor = ast.NodeVisitor()
visitor.visit(pt)
import redbaron
# Pares a FULL syntax tree that keeps blockcomments
baron = redbaron.RedBaron(sourcecode)
# fst = baron.fst()
node = baron.node_list[54] # NOQA
[n.type for n in baron.node_list]
# class SpecialVisitor(ast.NodeVisitor):
[docs]def parse_function_names(sourcecode, top_level=True, ignore_condition=1):
"""
Finds all function names in a file without importing it
Args:
sourcecode (str):
Returns:
list: func_names
CommandLine:
python -m utool.util_inspect parse_function_names
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> fpath = ut.util_inspect.__file__.replace('.pyc', '.py')
>>> #fpath = ut.truepath('~/code/bintrees/bintrees/avltree.py')
>>> sourcecode = ut.readfrom(fpath)
>>> func_names = parse_function_names(sourcecode)
>>> result = ('func_names = %s' % (ut.repr2(func_names),))
>>> print(result)
"""
import ast
import utool as ut
func_names = []
if six.PY2:
sourcecode = ut.ensure_unicode(sourcecode)
encoded = sourcecode.encode('utf8')
pt = ast.parse(encoded)
else:
pt = ast.parse(sourcecode)
class FuncVisitor(ast.NodeVisitor):
def __init__(self):
super(FuncVisitor, self).__init__()
self.condition_names = None
self.condition_id = -9001
self.in_condition_chain = False
def visit_If(self, node):
if ignore_condition:
return
# if ignore_conditional:
# return
# Ignore the main statement
# print('----')
# print('node.test = {!r}'.format(node.test))
# print('node.orelse = {!r}'.format(node.orelse))
if _node_is_main_if(node):
return
# if isinstance(node.orelse, ast.If):
# # THIS IS AN ELIF
# self.condition_id += 1
# self.in_condition_chain = True
# ast.NodeVisitor.generic_visit(self, node)
# self.in_condition_chain = False
# pass
# # TODO: where does else get parsed exactly?
# Reset the set of conditionals
# self.condition_id = 0
# self.condition_names = ut.ddict(list)
# self.in_condition_chain = True
ast.NodeVisitor.generic_visit(self, node)
# self.in_condition_chain = False
# if False:
# # IF THIS WAS AN ELSE:
# if self.condition_names is not None:
# # anything defined in all conditions is kosher
# from six.moves import reduce
# common_names = reduce(set.intersection,
# map(set, self.condition_names.values()))
# self.func_names.extend(common_names)
# self.condition_names = None
def visit_FunctionDef(self, node):
# if self.in_condition_chain and self.condition_names is not None:
# # dont immediately add things in conditions. Wait until we can
# # ensure which definitions are common in all conditions.
# self.condition_names[self.condition_id].append(node.name)
# else:
func_names.append(node.name)
if not top_level:
ast.NodeVisitor.generic_visit(self, node)
def visit_ClassDef(self, node):
if not top_level:
ast.NodeVisitor.generic_visit(self, node)
try:
FuncVisitor().visit(pt)
except Exception:
raise
pass
return func_names
def _node_is_main_if(node):
if isinstance(node.test, ast.Compare):
try:
if all(
[
isinstance(node.test.ops[0], ast.Eq),
node.test.left.id == '__name__',
node.test.comparators[0].s == '__main__',
]
):
return True
except Exception:
pass
return False
[docs]def parse_project_imports(dpath):
"""
dpath = ub.truepath('~/code/clab/clab')
Script:
>>> dpath = ut.get_argval('--dpath')
>>> parse_project_imports()
"""
import ubelt as ub
import glob
from os.path import join, exists
package_modules = set()
for fpath in glob.glob(join(dpath, '**/*.py'), recursive=True):
try:
sourcecode = ub.readfrom(fpath)
_, modules = ut.parse_import_names(sourcecode, False, fpath=fpath)
for mod in modules:
package_modules.add(mod.split('.')[0]) # just bases
if 'clab/live' in package_modules:
raise ValueError()
break
except SyntaxError:
print('encountered SyntaxError in fpath = {!r}'.format(fpath))
import warnings
import inspect
stdlibs = [dirname(warnings.__file__), dirname(inspect.__file__)]
def is_module_batteries_included(m):
if m in sys.builtin_module_names:
return True
for p in stdlibs:
if exists(join(p, m + '.py')):
return True
if exists(join(p, m)):
return True
used_modules = sorted(
[m for m in package_modules if not is_module_batteries_included(m)]
)
print('used_modules non-buildin modules = {}'.format(ub.repr2(used_modules)))
[docs]def parse_import_names(sourcecode, top_level=True, fpath=None, branch=False):
"""
Finds all function names in a file without importing it
Args:
sourcecode (str):
Returns:
list: func_names
CommandLine:
python -m utool.util_inspect parse_import_names
References:
https://stackoverflow.com/questions/20445733/how-to-tell-which-modules-have-been-imported-in-some-source-code
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> fpath = ut.util_inspect.__file__.replace('.pyc', '.py')
>>> #fpath = ut.truepath('~/code/bintrees/bintrees/avltree.py')
>>> sourcecode = ut.readfrom(fpath)
>>> func_names = parse_import_names(sourcecode)
>>> result = ('func_names = %s' % (ut.repr2(func_names),))
>>> print(result)
"""
import ast
import_names = []
if six.PY2:
import utool as ut
sourcecode = ut.ensure_unicode(sourcecode)
encoded = sourcecode.encode('utf8')
pt = ast.parse(encoded)
else:
pt = ast.parse(sourcecode)
modules = []
class ImportVisitor(ast.NodeVisitor):
def _parse_alias_list(self, aliases):
for alias in aliases:
if alias.asname is not None:
import_names.append(alias.asname)
else:
if '.' not in alias.name:
import_names.append(alias.name)
def visit_Import(self, node):
self._parse_alias_list(node.names)
self.generic_visit(node)
for alias in node.names:
modules.append(alias.name)
def visit_ImportFrom(self, node):
self._parse_alias_list(node.names)
self.generic_visit(node)
for alias in node.names:
prefix = ''
if node.level:
if fpath is not None:
from xdoctest import static_analysis as static
modparts = (
static.split_modpath(os.path.abspath(fpath))[1]
.replace('\\', '/')
.split('/')
)
parts = modparts[: -node.level]
# parts = os.path.split(static.split_modpath(os.path.abspath(fpath))[1])[:-node.level]
prefix = '.'.join(parts) + '.'
# prefix = '.'.join(os.path.split(fpath)[-node.level:]) + '.'
else:
prefix = '.' * node.level
# modules.append(node.level * '.' + node.module + '.' + alias.name)
# modules.append(prefix + node.module + '.' + alias.name)
modules.append(prefix + node.module)
def visit_FunctionDef(self, node):
# Ignore modules imported in functions
if not top_level:
self.generic_visit(node)
# ast.NodeVisitor.generic_visit(self, node)
def visit_ClassDef(self, node):
if not top_level:
self.generic_visit(node)
# ast.NodeVisitor.generic_visit(self, node)
def visit_If(self, node):
if not branch:
# TODO: determine how to figure out if a name is in all branches
if not _node_is_main_if(node):
# Ignore the main statement
self.generic_visit(node)
try:
ImportVisitor().visit(pt)
except Exception:
pass
return import_names, modules
[docs]def find_funcs_called_with_kwargs(sourcecode, target_kwargs_name='kwargs'):
r"""
Finds functions that are called with the keyword `kwargs` variable
CommandLine:
python3 -m utool.util_inspect find_funcs_called_with_kwargs
Example:
>>> # ENABLE_DOCTEST
>>> import utool as ut
>>> sourcecode = ut.codeblock(
'''
x, y = list(zip(*ut.ichunks(data, 2)))
somecall(arg1, arg2, arg3=4, **kwargs)
import sys
sys.badcall(**kwargs)
def foo():
bar(**kwargs)
ut.holymoly(**kwargs)
baz()
def biz(**kwargs):
foo2(**kwargs)
''')
>>> child_funcnamess = ut.find_funcs_called_with_kwargs(sourcecode)
>>> print('child_funcnamess = %r' % (child_funcnamess,))
>>> assert 'foo2' not in child_funcnamess, 'foo2 should not be found'
>>> assert 'bar' in child_funcnamess, 'bar should be found'
"""
import ast
sourcecode = 'from __future__ import print_function\n' + sourcecode
pt = ast.parse(sourcecode)
child_funcnamess = []
debug = False or VERYVERB_INSPECT
if debug:
print('\nInput:')
print('target_kwargs_name = %r' % (target_kwargs_name,))
print('\nSource:')
print(sourcecode)
import astor
print('\nParse:')
print(astor.dump(pt))
class KwargParseVisitor(ast.NodeVisitor):
"""
TODO: understand ut.update_existing and dict update
ie, know when kwargs is passed to these functions and
then look assume the object that was updated is a dictionary
and check wherever that is passed to kwargs as well.
"""
def visit_FunctionDef(self, node):
if debug:
print('\nVISIT FunctionDef node = %r' % (node,))
print('node.args.kwarg = %r' % (node.args.kwarg,))
if six.PY2:
kwarg_name = node.args.kwarg
else:
if node.args.kwarg is None:
kwarg_name = None
else:
kwarg_name = node.args.kwarg.arg
# import utool as ut
# ut.embed()
if kwarg_name != target_kwargs_name:
# target kwargs is still in scope
ast.NodeVisitor.generic_visit(self, node)
def visit_Call(self, node):
if debug:
print('\nVISIT Call node = %r' % (node,))
# print(ut.repr4(node.__dict__,))
if isinstance(node.func, ast.Attribute):
try:
funcname = node.func.value.id + '.' + node.func.attr
except AttributeError:
funcname = None
elif isinstance(node.func, ast.Name):
funcname = node.func.id
else:
raise NotImplementedError(
'do not know how to parse: node.func = %r' % (node.func,)
)
if six.PY2:
kwargs = node.kwargs
kwargs_name = None if kwargs is None else kwargs.id
if funcname is not None and kwargs_name == target_kwargs_name:
child_funcnamess.append(funcname)
if debug:
print('funcname = %r' % (funcname,))
print('kwargs_name = %r' % (kwargs_name,))
else:
if node.keywords:
for kwargs in node.keywords:
if kwargs.arg is None:
if hasattr(kwargs.value, 'id'):
kwargs_name = kwargs.value.id
if (
funcname is not None
and kwargs_name == target_kwargs_name
):
child_funcnamess.append(funcname)
if debug:
print('funcname = %r' % (funcname,))
print('kwargs_name = %r' % (kwargs_name,))
ast.NodeVisitor.generic_visit(self, node)
try:
KwargParseVisitor().visit(pt)
except Exception:
raise
pass
# import utool as ut
# if ut.SUPER_STRICT:
# raise
return child_funcnamess
# print('child_funcnamess = %r' % (child_funcnamess,))
[docs]def is_valid_python(code, reraise=True, ipy_magic_workaround=False):
"""
References:
http://stackoverflow.com/questions/23576681/python-check-syntax
"""
import ast
try:
if ipy_magic_workaround:
code = '\n'.join(
[
'pass' if re.match(r'\s*%[a-z]*', line) else line
for line in code.split('\n')
]
)
ast.parse(code)
except SyntaxError:
if reraise:
import utool as ut
print('Syntax Error')
ut.print_python_code(code)
raise
return False
return True
[docs]def parse_return_type(sourcecode):
r"""
parse_return_type
Args:
sourcecode (?):
Returns:
tuple: (return_type, return_name, return_header)
Ignore:
testcase
automated_helpers query_vsone_verified
CommandLine:
python -m utool.util_inspect parse_return_type
python -m utool.util_inspect --test-parse_return_type
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> sourcecode = ut.codeblock(
... 'def foo(tmp=False):\n'
... ' bar = True\n'
... ' return bar\n'
... )
>>> returninfo = parse_return_type(sourcecode)
>>> result = ut.repr2(returninfo)
>>> print(result)
('?', 'bar', 'Returns', '')
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> sourcecode = ut.codeblock(
... 'def foo(tmp=False):\n'
... ' return True\n'
... )
>>> returninfo = parse_return_type(sourcecode)
>>> result = ut.repr2(returninfo)
>>> print(result)
('bool', 'True', 'Returns', '')
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> sourcecode = ut.codeblock(
... 'def foo(tmp=False):\n'
... ' for i in range(2): \n'
... ' yield i\n'
... )
>>> returninfo = parse_return_type(sourcecode)
>>> result = ut.repr2(returninfo)
>>> print(result)
('?', 'i', 'Yields', '')
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> sourcecode = ut.codeblock(
... 'def foo(tmp=False):\n'
... ' if tmp is True:\n'
... ' return (True, False)\n'
... ' elif tmp is False:\n'
... ' return 1\n'
... ' else:\n'
... ' bar = baz()\n'
... ' return bar\n'
... )
>>> returninfo = parse_return_type(sourcecode)
>>> result = ut.repr2(returninfo)
>>> print(result)
('tuple', '(True, False)', 'Returns', '')
"""
import ast
return_type, return_name, return_header = (None, None, None)
if sourcecode is None:
return return_type, return_name, return_header, None
sourcecode = 'from __future__ import print_function\n' + sourcecode
pt = ast.parse(sourcecode)
import utool as ut
debug = ut.get_argflag('--debug-parse-return')
# debug = True
if debug:
import astor
print('\nSource:')
print(sourcecode)
print('\nParse:')
print(astor.dump(pt))
print('... starting')
def print_visit(type_, node):
if debug:
import utool as ut
print('+---')
print('\nVISIT %s node = %r' % (type_, node))
print('node.__dict__ = ' + ut.repr2(node.__dict__, nl=True))
print('L___')
def get_node_name_and_type(node):
if isinstance(node, ast.Tuple):
tupnode_list = node.elts
tupleid = '(%s)' % (
', '.join(
[str(get_node_name_and_type(tupnode)[1]) for tupnode in tupnode_list]
)
)
node_type = 'tuple'
node_name = tupleid
elif isinstance(node, ast.Dict):
node_type = 'dict'
node_name = None
elif isinstance(node, ast.Name):
node_name = node.id
node_type = '?'
if node_name in ['True', 'False']:
node_type = 'bool'
elif node_name == 'None':
node_type = 'None'
elif six.PY3 and isinstance(node, ast.NameConstant):
node_name = str(node.value)
node_type = '?'
if node_name in ['True', 'False', True, False]:
node_type = 'bool'
elif node_name in ['None', None]:
node_type = 'None'
else:
node_name = None
node_type = '?'
# node_type = 'ADD_TO_GET_NODE_NAME_AND_TYPE: ' + str(type(node.value))
return node_type, node_name
class ReturnVisitor(ast.NodeVisitor):
def init(self):
self.found_nodes = []
self.return_header = None
def visit_FunctionDef(self, node):
print_visit('FunctionDef', node)
# TODO: ignore subfunction return types
ast.NodeVisitor.generic_visit(self, node)
def visit_Return(self, node):
print_visit('Return', node)
ast.NodeVisitor.generic_visit(self, node)
return_value = node.value
print_visit('ReturnValue', return_value)
self.found_nodes.append(return_value)
self.return_header = 'Returns'
def visit_Yield(self, node):
print_visit('Yield', node)
ast.NodeVisitor.generic_visit(self, node)
return_value = node.value
print_visit('YieldValue', return_value)
self.found_nodes.append(return_value)
self.return_header = 'Yields'
try:
self = ReturnVisitor()
self.init()
self.visit(pt)
return_header = self.return_header
if len(self.found_nodes) > 0:
# hack rectify multiple return values
node = self.found_nodes[0]
return_type, return_name = get_node_name_and_type(node)
else:
return_name = None
return_type = 'None'
except Exception:
if debug:
raise
return_desc = ''
if return_type == '?':
tup = infer_arg_types_and_descriptions([return_name], [])
argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup
return_type = argtype_list[0]
return_desc = argdesc_list[0]
return return_type, return_name, return_header, return_desc
# def parse_return_type_OLD(sourcecode):
# import utool as ut
# import ast
# if ut.VERBOSE:
# print('[utool] parsing return types')
# if sourcecode is None:
# return_type, return_name, return_header = (None, None, None)
# return return_type, return_name, return_header, None
# #source_lines = sourcecode.splitlines()
# sourcecode = 'from __future__ import print_function\n' + sourcecode
# try:
# pt = ast.parse(sourcecode)
# except Exception:
# return_type, return_name, return_header = (None, None, None)
# #raise
# return return_type, return_name, return_header, None
# #print(sourcecode)
# #ut.printex(ex, 'Error Parsing')
# assert isinstance(pt, ast.Module), str(type(pt))
# Try = ast.Try if six.PY3 else ast.TryExcept
# def find_function_nodes(pt):
# function_nodes = []
# for node in pt.body:
# if isinstance(node, ast.FunctionDef):
# function_nodes.append(node)
# return function_nodes
# function_nodes = find_function_nodes(pt)
# assert len(function_nodes) == 1
# func_node = function_nodes[0]
# def find_return_node(node):
# if isinstance(node, list):
# candidates = []
# node_list = node
# for node in node_list:
# candidate = find_return_node(node)
# if candidate is not None:
# candidates.append(candidate)
# if len(candidates) > 0:
# return candidates[0]
# elif isinstance(node, (ast.Return, ast.Yield)):
# return node
# elif isinstance(node, (ast.If, Try)):
# return find_return_node(node.body)
# else:
# pass
# #print(type(node))
# if ut.VERBOSE:
# print('[utool] parsing return types')
# returnnode = find_return_node(func_node.body)
# # Check return or yeild
# if isinstance(returnnode, ast.Yield):
# return_header = 'Yeilds'
# elif isinstance(returnnode, ast.Return):
# return_header = 'Returns'
# else:
# return_header = None
# # Get more return info
# def get_node_name_and_type(node):
# node_name = None
# node_type = '?'
# if node is None:
# node_type = 'None'
# elif isinstance(node.value, ast.Tuple):
# tupnode_list = node.value.elts
# def get_tuple_membername(tupnode):
# if hasattr(tupnode, 'id'):
# return tupnode.id
# elif hasattr(tupnode, 'value'):
# return 'None'
# else:
# return 'None'
# pass
# tupleid = '(%s)' % (', '.join([str(get_tuple_membername(tupnode)) for tupnode in tupnode_list]))
# node_type = 'tuple'
# node_name = tupleid
# #node_name = ast.dump(node)
# elif isinstance(node.value, ast.Dict):
# node_type = 'dict'
# node_name = None
# elif isinstance(node.value, ast.Name):
# node_name = node.value.id
# if node_name == 'True':
# node_name = 'True'
# node_type = 'bool'
# else:
# #node_type = 'ADD_TO_GET_NODE_NAME_AND_TYPE: ' + str(type(node.value))
# node_type = '?'
# return node_type, node_name
# return_type, return_name = get_node_name_and_type(returnnode)
# if return_type == '?':
# tup = infer_arg_types_and_descriptions([return_name], [])
# argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup
# return_type = argtype_list[0]
# return_desc = argdesc_list[0]
# else:
# return_desc = ''
# return return_type, return_name, return_header, return_desc
[docs]def exec_func_src(
func,
globals_=None,
locals_=None,
key_list=None,
sentinal=None,
update=None,
keys=None,
verbose=False,
start=None,
stop=None,
):
"""
execs a func and returns requested local vars.
Does not modify globals unless update=True (or in IPython)
SeeAlso:
ut.execstr_funckw
"""
if keys is None:
keys = key_list
import utool as ut
sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True)
if update is None:
update = ut.inIPython()
if globals_ is None:
globals_ = ut.get_parent_frame().f_globals
if locals_ is None:
locals_ = ut.get_parent_frame().f_locals
if sentinal is not None:
sourcecode = ut.replace_between_tags(sourcecode, '', sentinal)
if start is not None or stop is not None:
sourcecode = '\n'.join(sourcecode.splitlines()[slice(start, stop)])
globals_new = globals_.copy()
if locals_ is not None:
globals_new.update(locals_)
# globals_new.update({k: v for k, v in locals_.items()
# if k not in globals_new})
orig_globals = globals_new.copy()
# six.exec_(sourcecode, globals_new, locals_)
if verbose:
print(ut.color_text(sourcecode, 'python'))
six.exec_(sourcecode, globals_new)
# Draw intermediate steps
if keys is None:
# return locals_
# Remove keys created in function execution
ut.delete_keys(globals_new, orig_globals.keys())
if update:
# update input globals?
globals_.update(globals_new)
# ~~ TODO autodetermine the keys from the function vars
return globals_new
else:
if update:
# update input globals?
globals_.update(globals_new)
# var_list = ut.dict_take(locals_, keys)
var_list = ut.dict_take(globals_new, keys)
return var_list
[docs]def exec_func_src2(
func, globals_=None, locals_=None, sentinal=None, verbose=False, start=None, stop=None
):
"""
execs a func and returns requested local vars.
Does not modify globals unless update=True (or in IPython)
SeeAlso:
ut.execstr_funckw
"""
import utool as ut
sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True)
if globals_ is None:
globals_ = ut.get_parent_frame().f_globals
if locals_ is None:
locals_ = ut.get_parent_frame().f_locals
if sentinal is not None:
sourcecode = ut.replace_between_tags(sourcecode, '', sentinal)
if start is not None or stop is not None:
sourcecode = '\n'.join(sourcecode.splitlines()[slice(start, stop)])
if verbose:
print(ut.color_text(sourcecode, 'python'))
# TODO: find the name of every variable that was assigned in the function
# and get it from the context
locals2_ = locals_.copy()
globals2_ = globals_.copy()
six.exec_(sourcecode, globals2_, locals2_)
return locals2_
[docs]def exec_func_src3(func, globals_, sentinal=None, verbose=False, start=None, stop=None):
"""
execs a func and returns requested local vars.
Does not modify globals unless update=True (or in IPython)
SeeAlso:
ut.execstr_funckw
"""
import utool as ut
sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True)
if sentinal is not None:
sourcecode = ut.replace_between_tags(sourcecode, '', sentinal)
if start is not None or stop is not None:
sourcecode = '\n'.join(sourcecode.splitlines()[slice(start, stop)])
if verbose:
print(ut.color_text(sourcecode, 'python'))
six.exec_(sourcecode, globals_)
[docs]def execstr_func_doctest(func, num=0, start_sentinal=None, end_sentinal=None):
r"""
execs a func doctest and returns requested local vars.
>>> from utool.util_inspect import * # NOQA
func = encoder.learn_threshold2
num = 0
start_sentinal = 'import wbia.plottool as pt'
end_sentinal = 'pnum\_ = pt.make_pnum_nextgen'
"""
import utool as ut
docsrc = ut.get_doctest_examples(func)[num][0]
lines = docsrc.split('\n')
if start_sentinal is None:
linex1 = 0
else:
linex1 = ut.where([x.startswith(start_sentinal) for x in lines])[0]
if end_sentinal is None:
linex2 = len(lines)
else:
linex2 = ut.where([x.startswith(end_sentinal) for x in lines])[0]
docsrc_part = '\n'.join(lines[linex1:linex2])
return docsrc_part
[docs]def exec_func_doctest(
func, start_sentinal=None, end_sentinal=None, num=0, globals_=None, locals_=None
):
r"""
execs a func doctest and returns requested local vars.
func = encoder.learn_threshold2
num = 0
start_sentinal = 'import wbia.plottool as pt'
end_sentinal = 'pnum\_ = pt.make_pnum_nextgen'
"""
import utool as ut
docsrc_part = execstr_func_doctest(func, num, start_sentinal, end_sentinal)
if globals_ is None:
globals_ = ut.get_parent_frame().f_globals
if locals_ is None:
locals_ = ut.get_parent_frame().f_locals
globals_new = globals_.copy()
if locals_ is not None:
globals_new.update(locals_)
print('EXEC PART')
print(ut.highlight_code(docsrc_part))
six.exec_(docsrc_part, globals_new)
[docs]def get_func_sourcecode(
func,
stripdef=False,
stripret=False,
strip_docstr=False,
strip_comments=False,
remove_linenums=None,
):
"""
wrapper around inspect.getsource but takes into account utool decorators
strip flags are very hacky as of now
Args:
func (function):
stripdef (bool):
stripret (bool): (default = False)
strip_docstr (bool): (default = False)
strip_comments (bool): (default = False)
remove_linenums (None): (default = None)
CommandLine:
python -m utool.util_inspect --test-get_func_sourcecode
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> # build test data
>>> func = get_func_sourcecode
>>> stripdef = True
>>> stripret = True
>>> sourcecode = get_func_sourcecode(func, stripdef)
>>> # verify results
>>> print(result)
"""
import utool as ut
# try:
inspect.linecache.clearcache() # HACK: fix inspect bug
sourcefile = inspect.getsourcefile(func)
# except IOError:
# sourcefile = None
if hasattr(func, '_utinfo'):
# if 'src' in func._utinfo:
# sourcecode = func._utinfo['src']
# else:
func2 = func._utinfo['orig_func']
sourcecode = get_func_sourcecode(func2)
elif sourcefile is not None and (sourcefile != '<string>'):
try_limit = 2
for num_tries in range(try_limit):
try:
# print(func)
sourcecode = inspect.getsource(func)
if not isinstance(sourcecode, six.text_type):
sourcecode = sourcecode.decode('utf-8')
# print(sourcecode)
except (IndexError, OSError, SyntaxError) as ex:
ut.printex(ex, 'Error getting source', keys=['sourcefile', 'func'])
if False:
# VERY HACK: try to reload the module and get a redefined
# version of the function
import imp
modname = get_func_modname(func)
funcname = ut.get_funcname(func)
module = sys.modules[modname]
# TODO: ut.reload_module()
module = imp.reload(module)
func = module.__dict__[funcname]
else:
# Fix inspect bug in python2.7
inspect.linecache.clearcache()
if num_tries + 1 != try_limit:
tries_left = try_limit - num_tries - 1
print('Attempting %d more time(s)' % (tries_left))
else:
raise
else:
sourcecode = None
# orig_source = sourcecode
# print(orig_source)
if stripdef:
# hacky
sourcecode = ut.unindent(sourcecode)
# sourcecode = ut.unindent(ut.regex_replace('def [^)]*\\):\n', '', sourcecode))
# nodef_source = ut.regex_replace('def [^:]*\\):\n', '', sourcecode)
regex_decor = '^@.' + ut.REGEX_NONGREEDY
regex_defline = '^def [^:]*\\):\n'
patern = '(' + regex_decor + ')?' + regex_defline
nodef_source = ut.regex_replace(patern, '', sourcecode)
sourcecode = ut.unindent(nodef_source)
# print(sourcecode)
pass
if stripret:
r"""\s is a whitespace char"""
return_ = ut.named_field('return', 'return .*$')
prereturn = ut.named_field('prereturn', r'^\s*')
return_bref = ut.bref_field('return')
prereturn_bref = ut.bref_field('prereturn')
regex = prereturn + return_
repl = prereturn_bref + 'pass # ' + return_bref
# import re
# print(re.search(regex, sourcecode, flags=re.MULTILINE ))
# print(re.search('return', sourcecode, flags=re.MULTILINE | re.DOTALL ))
# print(re.search(regex, sourcecode))
sourcecode_ = re.sub(regex, repl, sourcecode, flags=re.MULTILINE)
# print(sourcecode_)
sourcecode = sourcecode_
pass
if strip_docstr or strip_comments:
# pip install pyminifier
# References: http://code.activestate.com/recipes/576704/
# from pyminifier import minification, token_utils
def remove_docstrings_or_comments(source):
"""
TODO: commit clean version to pyminifier
"""
import tokenize
from six.moves import StringIO
io_obj = StringIO(source)
out = ''
prev_toktype = tokenize.INDENT
last_lineno = -1
last_col = 0
for tok in tokenize.generate_tokens(io_obj.readline):
token_type = tok[0]
token_string = tok[1]
start_line, start_col = tok[2]
end_line, end_col = tok[3]
if start_line > last_lineno:
last_col = 0
if start_col > last_col:
out += ' ' * (start_col - last_col)
# Remove comments:
if strip_comments and token_type == tokenize.COMMENT:
pass
elif strip_docstr and token_type == tokenize.STRING:
if prev_toktype != tokenize.INDENT:
# This is likely a docstring; double-check we're not inside an operator:
if prev_toktype != tokenize.NEWLINE:
if start_col > 0:
out += token_string
else:
out += token_string
prev_toktype = token_type
last_col = end_col
last_lineno = end_line
return out
sourcecode = remove_docstrings_or_comments(sourcecode)
# sourcecode = minification.remove_comments_and_docstrings(sourcecode)
# tokens = token_utils.listified_tokenizer(sourcecode)
# minification.remove_comments(tokens)
# minification.remove_docstrings(tokens)
# token_utils.untokenize(tokens)
if remove_linenums is not None:
source_lines = sourcecode.strip('\n').split('\n')
ut.delete_items_by_index(source_lines, remove_linenums)
sourcecode = '\n'.join(source_lines)
return sourcecode
# else:
# return get_func_sourcecode(func._utinfo['src'])
[docs]def get_unbound_args(argspec):
try:
args = argspec.args
except Exception:
func = argspec
argspec = get_func_argspec(func)
args = argspec.args
args = argspec.args
defaults = argspec.defaults
if defaults is not None:
kwpos = len(args) - len(defaults)
unbound_args = args[:kwpos]
else:
unbound_args = args
return unbound_args
[docs]def get_func_argspec(func):
"""
wrapper around inspect.getfullargspec but takes into account utool decorators
"""
if hasattr(func, '_utinfo'):
argspec = func._utinfo['orig_argspec']
return argspec
if isinstance(func, property):
func = func.fget
argspec = inspect.getfullargspec(func)
return argspec
[docs]def get_kwargs(func):
r"""
Args:
func (function):
Returns:
tuple: keys, is_arbitrary
keys (list): kwargs keys
is_arbitrary (bool): has generic \*\*kwargs
CommandLine:
python -m utool.util_inspect --test-get_kwargs
Ignore:
>>> def func1(a, b, c):
>>> pass
>>> def func2(a, b, c, *args):
>>> pass
>>> def func3(a, b, c, *args, **kwargs):
>>> pass
>>> def func4(a, b=1, c=2):
>>> pass
>>> def func5(a, b=1, c=2, *args):
>>> pass
>>> def func6(a, b=1, c=2, **kwargs):
>>> pass
>>> def func7(a, b=1, c=2, *args, **kwargs):
>>> pass
>>> for func in [locals()['func' + str(x)] for x in range(1, 8)]:
>>> print(inspect.getfullargspec(func))
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> # build test data
>>> func = '?'
>>> result = get_kwargs(func)
>>> # verify results
>>> print(result)
"""
# if argspec.keywords is None:
import utool as ut
argspec = inspect.getfullargspec(func)
if argspec.defaults is not None:
num_args = len(argspec.args)
num_keys = len(argspec.defaults)
keys = ut.take(argspec.args, range(num_args - num_keys, num_args))
else:
keys = []
is_arbitrary = argspec.varkw is not None
RECURSIVE = False
if RECURSIVE and argspec.varkw is not None:
pass
# TODO: look inside function at the functions that the kwargs object is being
# passed to
return keys, is_arbitrary
[docs]def lookup_attribute_chain(attrname, namespace):
"""
Ignore:
>>> attrname = funcname
>>> namespace = mod.__dict__
>>> import utool as ut
>>> globals_ = ut.util_inspect.__dict__
>>> attrname = 'KWReg.print_defaultkw'
"""
# subdict = meta_util_six.get_funcglobals(root_func)
subtup = attrname.split('.')
subdict = namespace
for attr in subtup[:-1]:
subdict = subdict[attr].__dict__
leaf_name = subtup[-1]
leaf_attr = subdict[leaf_name]
return leaf_attr
[docs]def recursive_parse_kwargs(root_func, path_=None, verbose=None):
"""
recursive kwargs parser
TODO: rectify with others
FIXME: if docstr indentation is off, this fails
SeeAlso:
argparse_funckw
recursive_parse_kwargs
parse_kwarg_keys
parse_func_kwarg_keys
get_func_kwargs
Args:
root_func (function): live python function
path_ (None): (default = None)
Returns:
list:
CommandLine:
python -m utool.util_inspect recursive_parse_kwargs:0
python -m utool.util_inspect recursive_parse_kwargs:0 --verbinspect
python -m utool.util_inspect recursive_parse_kwargs:1
python -m utool.util_inspect recursive_parse_kwargs:2 --mod vtool --func ScoreNormalizer.visualize
python -m utool.util_inspect recursive_parse_kwargs:2 --mod wbia.viz.viz_matches --func show_name_matches --verbinspect
python -m utool.util_inspect recursive_parse_kwargs:2 --mod wbia.expt.experiment_drawing --func draw_rank_cmc --verbinspect
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> root_func = iter_module_doctestable
>>> path_ = None
>>> result = ut.repr2(recursive_parse_kwargs(root_func), nl=1)
>>> print(result)
[
('include_funcs', True),
('include_classes', True),
('include_methods', True),
('include_builtin', True),
('include_inherited', False),
('debug_key', None),
]
Example:
>>> # xdoctest: +REQUIRES(module:wbia)
>>> from utool.util_inspect import * # NOQA
>>> from wbia.algo.hots import chip_match
>>> import utool as ut
>>> recursive_parse_kwargs(chip_match.ChipMatch.show_ranked_matches)
>>> recursive_parse_kwargs(chip_match.ChipMatch)
import wbia
import utool as ut
ibs = wbia.opendb(defaultdb='testdb1')
kwkeys1 = ibs.parse_annot_stats_filter_kws()
ut.recursive_parse_kwargs(ibs.get_annotconfig_stats, verbose=1)
kwkeys2 = list(ut.recursive_parse_kwargs(ibs.get_annotconfig_stats).keys())
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> modname = ut.get_argval('--mod', type_=str, default='plottool')
>>> funcname = ut.get_argval('--func', type_=str, default='draw_histogram')
>>> mod = ut.import_modname(modname)
>>> root_func = lookup_attribute_chain(funcname, mod.__dict__)
>>> path_ = None
>>> parsed = recursive_parse_kwargs(root_func)
>>> flags = ut.unique_flags(ut.take_column(parsed, 0))
>>> unique = ut.compress(parsed, flags)
>>> print('parsed = %s' % (ut.repr4(parsed),))
>>> print('unique = %s' % (ut.repr4(unique),))
"""
if verbose is None:
verbose = VERBOSE_INSPECT
if verbose:
print('[inspect] recursive parse kwargs root_func = %r ' % (root_func,))
import utool as ut
if path_ is None:
path_ = []
if root_func in path_:
if verbose:
print('[inspect] Encountered cycle. returning')
return []
path_.append(root_func)
spec = ut.get_func_argspec(root_func)
# ADD MORE
kwargs_list = []
found_explicit = list(ut.get_kwdefaults(root_func, parse_source=False).items())
if verbose:
print('[inspect] * Found explicit %r' % (found_explicit,))
# kwargs_list = [(kw,) for kw in ut.get_kwargs(root_func)[0]]
sourcecode = ut.get_func_sourcecode(root_func, strip_docstr=True, stripdef=True)
sourcecode1 = ut.get_func_sourcecode(root_func, strip_docstr=True, stripdef=False)
found_implicit = ut.parse_kwarg_keys(sourcecode1, spec.varkw, with_vals=True)
if verbose:
print('[inspect] * Found found_implicit %r' % (found_implicit,))
kwargs_list = found_explicit + found_implicit
def hack_lookup_mod_attrs(attr):
# HACKS TODO: have find_funcs_called_with_kwargs infer an attribute is a
# module / function / type. In the module case, we can import it and
# look it up. Maybe args, or returns can help infer type. Maybe just
# register some known varnames. Maybe jedi has some better way to do
# this.
if attr == 'ut':
subdict = ut.__dict__
elif attr == 'pt':
import wbia.plottool as pt
subdict = pt.__dict__
else:
subdict = None
return subdict
def resolve_attr_subfunc(subfunc_name):
# look up attriute chain
# subdict = root_func.func_globals
subdict = meta_util_six.get_funcglobals(root_func)
subtup = subfunc_name.split('.')
try:
subdict = lookup_attribute_chain(subfunc_name, subdict)
if ut.is_func_or_method(subdict):
# Was subdict supposed to be named something else here?
subfunc = subdict
return subfunc
except (KeyError, TypeError):
for attr in subtup[:-1]:
try:
subdict = subdict[attr].__dict__
except (KeyError, TypeError):
# limited support for class lookup
if ut.is_method(root_func) and spec.args[0] == attr:
if six.PY2:
subdict = root_func.im_class.__dict__
else:
subdict = root_func.__class__.__dict__
else:
# FIXME TODO lookup_attribute_chain
subdict = hack_lookup_mod_attrs(attr)
if subdict is None:
print('Unable to find attribute of attr=%r' % (attr,))
if ut.SUPER_STRICT:
raise
if subdict is not None:
attr_name = subtup[-1]
subfunc = subdict[attr_name]
else:
subfunc = None
return subfunc
def check_subfunc_name(subfunc_name):
if isinstance(subfunc_name, tuple) or '.' in subfunc_name:
subfunc = resolve_attr_subfunc(subfunc_name)
else:
# try to directly take func from globals
func_globals = meta_util_six.get_funcglobals(root_func)
try:
subfunc = func_globals[subfunc_name]
except KeyError:
print(
'Unable to find function definition subfunc_name=%r' % (subfunc_name,)
)
if ut.SUPER_STRICT:
raise
subfunc = None
if subfunc is not None:
subkw_list = recursive_parse_kwargs(subfunc, path_, verbose=verbose)
new_subkw = subkw_list
# have_keys = set(ut.take_column(kwargs_list, 0))
# new_subkw = [item for item in subkw_list
# if item[0] not in have_keys]
else:
new_subkw = []
return new_subkw
if spec.varkw is not None:
if verbose:
print('[inspect] Checking spec.varkw=%r' % (spec.varkw,))
subfunc_name_list = ut.find_funcs_called_with_kwargs(sourcecode, spec.varkw)
if verbose:
print(
'[inspect] Checking subfunc_name_list with len {}'.format(
len(subfunc_name_list)
)
)
for subfunc_name in subfunc_name_list:
try:
new_subkw = check_subfunc_name(subfunc_name)
if verbose:
print('[inspect] * Found %r' % (new_subkw,))
kwargs_list.extend(new_subkw)
except (TypeError, Exception):
print(
'warning: unable to recursivley parse type of : %r' % (subfunc_name,)
)
return kwargs_list
[docs]def get_funckw(func, recursive=True):
import utool as ut
funckw_ = ut.get_func_kwargs(func, recursive=recursive)
# if recursive:
# funckw_.update(dict(ut.recursive_parse_kwargs(func)))
return funckw_
[docs]def parse_func_kwarg_keys(func, with_vals=False):
"""hacky inference of kwargs keys
SeeAlso:
argparse_funckw
recursive_parse_kwargs
parse_kwarg_keys
parse_func_kwarg_keys
get_func_kwargs
"""
sourcecode = get_func_sourcecode(func, strip_docstr=True, strip_comments=True)
kwkeys = parse_kwarg_keys(sourcecode, with_vals=with_vals)
# ut.get_func_kwargs TODO
return kwkeys
[docs]def get_func_kwargs(func, recursive=True):
"""
func = wbia.run_experiment
SeeAlso:
argparse_funckw
recursive_parse_kwargs
parse_kwarg_keys
parse_func_kwarg_keys
get_func_kwargs
"""
import utool as ut
argspec = ut.get_func_argspec(func)
if argspec.defaults is None:
header_kw = {}
else:
header_kw = dict(zip(argspec.args[::-1], argspec.defaults[::-1]))
if argspec.varkw is not None:
header_kw.update(dict(ut.recursive_parse_kwargs(func)))
return header_kw
[docs]def parse_kwarg_keys(source, keywords='kwargs', with_vals=False):
r"""
Parses the source code to find keys used by the `**kwargs` keywords
dictionary variable. if `with_vals` is True, we also attempt to infer the
default values.
Args:
source (str):
Returns:
list: kwarg_keys
CommandLine:
python -m utool.util_inspect parse_kwarg_keys
python -m utool.util_inspect parse_kwarg_keys
SeeAlso:
argparse_funckw
recursive_parse_kwargs
parse_kwarg_keys
parse_func_kwarg_keys
get_func_kwargs
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> source = (
>>> "\n x = 'hidden_x'"
>>> "\n y = 3 # hidden val"
>>> "\n kwargs.get(x, y)"
>>> "\n kwargs.get('foo', None)\n kwargs.pop('bar', 3)"
>>> "\n kwargs.pop('str', '3fd')\n kwargs.pop('str', '3f\\'d')"
>>> "\n \"kwargs.get('baz', None)\"\n kwargs['foo2']"
>>> "\n #kwargs.get('biz', None)\""
>>> "\n kwargs['bloop']"
>>> "\n x = 'bop' in kwargs"
>>> )
>>> print('source = %s\n' % (source,))
>>> ut.exec_funckw(parse_kwarg_keys, globals())
>>> with_vals = True
>>> kwarg_items = parse_kwarg_keys(source, with_vals=with_vals)
>>> result = ('kwarg_items = %s' % (ut.repr2(kwarg_items, nl=1),))
>>> kwarg_keys = ut.take_column(kwarg_items, 0)
>>> assert 'baz' not in kwarg_keys
>>> assert 'foo' in kwarg_keys
>>> assert 'bloop' in kwarg_keys
>>> assert 'bop' not in kwarg_keys
>>> print(result)
kwarg_items = [
('foo', None),
('bar', 3),
('str', '3fd'),
('str', "3f'd"),
('foo2', None),
('bloop', None),
]
"""
import utool as ut
import ast
sourcecode = (
'from __future__ import print_function, unicode_literals\n' + ut.unindent(source)
)
pt = ast.parse(sourcecode)
kwargs_items = []
debug = VERYVERB_INSPECT
target_kwargs_name = keywords
if debug:
import astor
print('\nInput:')
print('target_kwargs_name = %r' % (target_kwargs_name,))
print('\nSource:')
print(sourcecode)
print('\nParse:')
print(astor.dump(pt))
class KwargParseVisitor(ast.NodeVisitor):
"""
TODO: understand ut.update_existing and dict update
ie, know when kwargs is passed to these functions and
then look assume the object that was updated is a dictionary
and check wherever that is passed to kwargs as well.
Other visit_<classname> values:
http://greentreesnakes.readthedocs.io/en/latest/nodes.html
"""
def __init__(self):
super(KwargParseVisitor, self).__init__()
self.const_lookup = {}
self.first = True
def visit_FunctionDef(self, node):
if debug:
print('VISIT FunctionDef node = %r' % (node,))
# print('node.args.kwarg = %r' % (node.args.kwarg,))
if six.PY2:
kwarg_name = node.args.kwarg
else:
if node.args.kwarg is None:
kwarg_name = None
else:
kwarg_name = node.args.kwarg.arg
# Record any constants defined in function definitions
defaults_vals = node.args.defaults
offset = len(node.args.args) - len(defaults_vals)
default_keys = node.args.args[offset:]
for kwname, kwval in zip(default_keys, defaults_vals):
# try:
if six.PY2:
if isinstance(kwval, ast.Name):
val = eval(kwval.id, {}, {})
self.const_lookup[kwname.id] = val
else:
if isinstance(kwval, ast.NameConstant):
val = kwval.value
self.const_lookup[kwname.arg] = val
# except Exception:
# pass
if self.first or kwarg_name != target_kwargs_name:
# target kwargs is still in scope
ast.NodeVisitor.generic_visit(self, node)
# always visit the first function
self.first = False
def visit_Subscript(self, node):
if debug:
print('VISIT SUBSCRIPT node = %r' % (node,))
# print(ut.repr4(node.__dict__,))
if isinstance(node.value, ast.Name):
if node.value.id == target_kwargs_name:
if isinstance(node.slice, ast.Index):
index = node.slice
key = index.value
if isinstance(key, ast.Str):
# item = (key.s, None)
item = (key.s, None)
kwargs_items.append(item)
@staticmethod
def _eval_bool_op(val):
# Can we handle this more intelligently?
val_value = None
if isinstance(val.op, ast.Or):
if any(
[
isinstance(x, ast.NameConstant) and x.value is True
for x in val.values
]
):
val_value = True
elif isinstance(val.op, ast.And):
if any(
[
isinstance(x, ast.NameConstant) and x.value is False
for x in val.values
]
):
val_value = False
return val_value
def visit_Call(self, node):
if debug:
print('VISIT Call node = %r' % (node,))
# print(ut.repr4(node.__dict__,))
if isinstance(node.func, ast.Attribute):
try:
objname = node.func.value.id
except AttributeError:
return
methodname = node.func.attr
# funcname = objname + '.' + methodname
if objname == target_kwargs_name and methodname in {'get', 'pop'}:
args = node.args
if len(args) == 2:
key, val = args
if isinstance(key, ast.Name):
# TODO lookup constant
pass
elif isinstance(key, ast.Str):
key_value = key.s
val_value = None # ut.NoParam
if isinstance(val, ast.Str):
val_value = val.s
elif isinstance(val, ast.Num):
val_value = val.n
elif isinstance(val, ast.Name):
if val.id == 'None':
val_value = None
else:
val_value = self.const_lookup.get(val.id, None)
# val_value = 'TODO lookup const'
# TODO: lookup constants?
pass
elif six.PY3:
if isinstance(val, ast.NameConstant):
val_value = val.value
elif isinstance(val, ast.Call):
val_value = None
elif isinstance(val, ast.BoolOp):
val_value = self._eval_bool_op(val)
elif isinstance(val, ast.Dict):
if len(val.keys) == 0:
val_value = {}
else:
val_value = {}
# val_value = callable
else:
print(
'Warning: util_inspect doent know how to parse {}'.format(
repr(val)
)
)
item = (key_value, val_value)
kwargs_items.append(item)
ast.NodeVisitor.generic_visit(self, node)
try:
KwargParseVisitor().visit(pt)
except Exception:
raise
pass
if with_vals:
return kwargs_items
else:
return ut.take_column(kwargs_items, 0)
[docs]class KWReg(object):
"""
Helper to register keywords for complex keyword parsers
"""
def __init__(kwreg, enabled=False):
kwreg.keys = []
kwreg.defaults = []
kwreg.enabled = enabled
def __call__(kwreg, key, default):
if kwreg.enabled:
kwreg.keys.append(key)
kwreg.defaults.append(default)
return key, default
@property
def defaultkw(kwreg):
return dict(zip(kwreg.keys, kwreg.defaults))
[docs] def print_defaultkw(kwreg):
print(ut.repr4(kwreg.defaultkw))
[docs]def get_instance_attrnames(obj, default=True, **kwargs):
cls = obj.__class__
out = []
for a in dir(cls):
unbound_attr = getattr(cls, a, None)
if kwargs.get('with_properties', default) and isinstance(unbound_attr, property):
out.append(a)
if kwargs.get('with_methods', default) and isinstance(
unbound_attr, types.MethodType
):
out.append(a)
return out
[docs]def argparse_funckw(func, defaults={}, **kwargs):
"""
allows kwargs to be specified on the commandline from testfuncs
Args:
func (function):
Kwargs:
lbl, verbose, only_specified, force_keys, type_hint, alias_dict
Returns:
dict: funckw
CommandLine:
python -m utool.util_inspect argparse_funckw
SeeAlso:
exec_funckw
recursive_parse_kwargs
parse_kwarg_keys
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> func = get_instance_attrnames
>>> funckw = argparse_funckw(func)
>>> result = ('funckw = %s' % (ut.repr3(funckw),))
>>> print(result)
funckw = {
'default': True,
'with_methods': True,
'with_properties': True,
}
"""
import utool as ut
funckw_ = ut.get_funckw(func, recursive=True)
funckw_.update(defaults)
funckw = ut.argparse_dict(funckw_, **kwargs)
return funckw
[docs]def infer_function_info(func):
r"""
Infers information for make_default_docstr
# TODO: Interleave old documentation with new documentation
Args:
func (function): live python function
CommandLine:
python -m utool --tf infer_function_info:0
python -m utool --tf infer_function_info:1 --funcname=wbia_cnn.models.siam.ignore_hardest_cases
Ignore:
>>> # ENABLE_DOCTEST
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> func = ut.infer_function_info
>>> #func = ut.Timer.tic
>>> func = ut.make_default_docstr
>>> funcinfo = infer_function_info(func)
>>> result = ut.repr4(funcinfo.__dict__)
>>> print(result)
Ignore:
>>> # DISABLE_DOCTEST
>>> # SCRIPT
>>> from utool.util_inspect import * # NOQA
>>> import utool as ut
>>> funcname = ut.get_argval('--funcname')
>>> # Parse out custom function
>>> modname = '.'.join(funcname.split('.')[0:-1])
>>> script = 'import {modname}\nfunc = {funcname}'.format(
>>> modname=modname, funcname=funcname)
>>> globals_, locals_ = {}, {}
>>> exec(script, globals_, locals_)
>>> func = locals_['func']
>>> funcinfo = infer_function_info(func)
>>> result = ut.repr4(funcinfo.__dict__)
>>> print(result)
"""
import utool as ut
import re
# TODO: allow a jedi argument
if False:
from jedi.evaluate import docstrings
script = func.script
argname_list = [p.name.value for p in func.params]
argtype_list = [
docstrings.follow_param(script._evaluator, p) for p in func.params
]
if isinstance(func, property):
func = func.fget
try:
doc_shortdesc = ''
doc_longdesc = ''
known_arginfo = ut.ddict(dict)
current_doc = inspect.getdoc(func)
docstr_blocks = ut.parse_docblocks_from_docstr(current_doc)
docblock_types = ut.take_column(docstr_blocks, 0)
docblock_types = [
re.sub('Example[0-9]', 'Example', type_) for type_ in docblock_types
]
docblock_dict = ut.group_items(docstr_blocks, docblock_types)
if '' in docblock_dict:
docheaders = docblock_dict['']
docheaders_lines = ut.take_column(docheaders, 1)
docheaders_order = ut.take_column(docheaders, 2)
docheaders_lines = ut.sortedby(docheaders_lines, docheaders_order)
doc_shortdesc = '\n'.join(docheaders_lines)
if 'Args' in docblock_dict:
argblocks = docblock_dict['Args']
if len(argblocks) != 1:
print('Warning: should only be one args block')
else:
argblock = argblocks[0][1]
assert argblock.startswith('Args:\n')
argsblock_ = argblock[len('Args:\n') :]
arglines = re.split(r'^ \b', argsblock_, flags=re.MULTILINE)
arglines = [line for line in arglines if len(line) > 0]
esc = re.escape
def escparen(pat):
return esc('(') + pat + esc(')')
argname = ut.named_field('argname', ut.REGEX_VARNAME)
argtype_ = ut.named_field('argtype', '.' + ut.REGEX_NONGREEDY)
argtype = escparen(argtype_)
argdesc = ut.named_field('argdesc', '.*')
WS = ut.REGEX_WHITESPACE
argpattern = WS + argname + WS + argtype + WS + ':' + WS + argdesc
for argline in arglines:
m = re.match(argpattern, argline, flags=re.MULTILINE | re.DOTALL)
try:
groupdict_ = m.groupdict()
except Exception:
print('---')
print('argline = \n%s' % (argline,))
print('---')
raise Exception('Unable to parse argline=%s' % (argline,))
# print('groupdict_ = %s' % (ut.repr4(groupdict_),))
argname = groupdict_['argname']
known_arginfo[argname]['argdesc'] = groupdict_['argdesc'].rstrip('\n')
# TODO: record these in a file for future reference
# and potential guessing
if groupdict_['argtype'] != '?':
known_arginfo[argname]['argtype'] = groupdict_['argtype']
is_class = isinstance(func, six.class_types)
needs_surround = current_doc is None or len(current_doc) == 0
if is_class:
argfunc = func.__init__
else:
argfunc = func
argspec = ut.get_func_argspec(argfunc)
(argname_list, varargs, varkw, defaults, _, _, _) = argspec
# See util_inspect
tup = ut.infer_arg_types_and_descriptions(argname_list, defaults)
argtype_list, argdesc_list, argdefault_list, hasdefault_list = tup
# Put in user parsed info
for index, argname in enumerate(argname_list):
if argname in known_arginfo:
arginfo = known_arginfo[argname]
if 'argdesc' in arginfo:
argdesc_list[index] = arginfo['argdesc']
if 'argtype' in arginfo:
argtype_list[index] = arginfo['argtype']
if not is_class:
# Move source down to base indentation, but remember original indentation
sourcecode = get_func_sourcecode(func)
# kwarg_keys = ut.parse_kwarg_keys(sourcecode)
kwarg_items = ut.recursive_parse_kwargs(func)
flags = ut.unique_flags(ut.take_column(kwarg_items, 0))
kwarg_items = ut.compress(kwarg_items, flags)
kwarg_keys = ut.take_column(kwarg_items, 0)
# kwarg_keys = ut.unique_ordered(kwarg_keys)
kwarg_keys = ut.setdiff_ordered(kwarg_keys, argname_list)
else:
sourcecode = None
kwarg_keys = []
if sourcecode is not None:
num_indent = ut.get_indentation(sourcecode)
sourcecode = ut.unindent(sourcecode)
returninfo = ut.parse_return_type(sourcecode)
else:
num_indent = 0
returninfo = None, None, None, ''
return_type, return_name, return_header, return_desc = returninfo
modname = func.__module__
funcname = ut.get_funcname(func)
except Exception as ex:
# print('dealing with infer function error')
# print('has utinfo? ' + str(hasattr(func, '_utinfo')))
# sourcefile = inspect.getsourcefile(func) # NOQA
ut.printex(
ex,
'Error Infering Function Info',
keys=['func', 'sourcefile', 'sourcecode', 'argspec'],
tb=True,
)
raise
class FunctionInfo(object):
def __init__(self):
pass
funcinfo = FunctionInfo()
funcinfo.needs_surround = needs_surround
funcinfo.argname_list = argname_list
funcinfo.argtype_list = argtype_list
funcinfo.argdesc_list = argdesc_list
funcinfo.argdefault_list = argdefault_list
funcinfo.hasdefault_list = hasdefault_list
funcinfo.kwarg_keys = kwarg_keys
# if new
funcinfo.va_name = varargs
funcinfo.kw_name = varkw
funcinfo.kw_keys = kwarg_keys
# else
funcinfo.varargs = varargs
funcinfo.varkw = varkw
# fi
funcinfo.defaults = defaults
funcinfo.num_indent = num_indent
funcinfo.return_type = return_type
funcinfo.return_name = return_name
funcinfo.return_header = return_header
funcinfo.return_desc = return_desc
funcinfo.modname = modname
funcinfo.funcname = funcname
funcinfo.doc_shortdesc = doc_shortdesc
funcinfo.doc_longdesc = doc_longdesc
funcinfo.ismethod = hasattr(func, 'im_class')
return funcinfo
if __name__ == '__main__':
"""
CommandLine:
python -m utool.util_inspect --enableall
python -m utool.util_inspect --allexamples
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()