Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/venusian/__init__.py : 60%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from inspect import getmembers, getmro, isclass
2from pkgutil import iter_modules
3import sys
5from venusian.advice import getFrameInfo
7ATTACH_ATTR = "__venusian_callbacks__"
8LIFTONLY_ATTR = "__venusian_liftonly_callbacks__"
11class Scanner(object):
12 def __init__(self, **kw):
13 self.__dict__.update(kw)
15 def scan(self, package, categories=None, onerror=None, ignore=None):
16 """ Scan a Python package and any of its subpackages. All
17 top-level objects will be considered; those marked with
18 venusian callback attributes related to ``category`` will be
19 processed.
21 The ``package`` argument should be a reference to a Python
22 package or module object.
24 The ``categories`` argument should be sequence of Venusian
25 callback categories (each category usually a string) or the
26 special value ``None`` which means all Venusian callback
27 categories. The default is ``None``.
29 The ``onerror`` argument should either be ``None`` or a callback
30 function which behaves the same way as the ``onerror`` callback
31 function described in
32 http://docs.python.org/library/pkgutil.html#pkgutil.walk_packages .
33 By default, during a scan, Venusian will propagate all errors that
34 happen during its code importing process, including
35 :exc:`ImportError`. If you use a custom ``onerror`` callback, you
36 can change this behavior.
38 Here's an example ``onerror`` callback that ignores
39 :exc:`ImportError`::
41 import sys
42 def onerror(name):
43 if not issubclass(sys.exc_info()[0], ImportError):
44 raise # reraise the last exception
46 The ``name`` passed to ``onerror`` is the module or package dotted
47 name that could not be imported due to an exception.
49 .. versionadded:: 1.0
50 the ``onerror`` callback
52 The ``ignore`` argument allows you to ignore certain modules,
53 packages, or global objects during a scan. It should be a sequence
54 containing strings and/or callables that will be used to match
55 against the full dotted name of each object encountered during a
56 scan. The sequence can contain any of these three types of objects:
58 - A string representing a full dotted name. To name an object by
59 dotted name, use a string representing the full dotted name. For
60 example, if you want to ignore the ``my.package`` package *and any
61 of its subobjects or subpackages* during the scan, pass
62 ``ignore=['my.package']``.
64 - A string representing a relative dotted name. To name an object
65 relative to the ``package`` passed to this method, use a string
66 beginning with a dot. For example, if the ``package`` you've
67 passed is imported as ``my.package``, and you pass
68 ``ignore=['.mymodule']``, the ``my.package.mymodule`` mymodule *and
69 any of its subobjects or subpackages* will be omitted during scan
70 processing.
72 - A callable that accepts a full dotted name string of an object as
73 its single positional argument and returns ``True`` or ``False``.
74 For example, if you want to skip all packages, modules, and global
75 objects with a full dotted path that ends with the word "tests", you
76 can use ``ignore=[re.compile('tests$').search]``. If the callable
77 returns ``True`` (or anything else truthy), the object is ignored,
78 if it returns ``False`` (or anything else falsy) the object is not
79 ignored. *Note that unlike string matches, ignores that use a
80 callable don't cause submodules and subobjects of a module or
81 package represented by a dotted name to also be ignored, they match
82 individual objects found during a scan, including packages,
83 modules, and global objects*.
85 You can mix and match the three types of strings in the list. For
86 example, if the package being scanned is ``my``,
87 ``ignore=['my.package', '.someothermodule',
88 re.compile('tests$').search]`` would cause ``my.package`` (and all
89 its submodules and subobjects) to be ignored, ``my.someothermodule``
90 to be ignored, and any modules, packages, or global objects found
91 during the scan that have a full dotted name that ends with the word
92 ``tests`` to be ignored.
94 Note that packages and modules matched by any ignore in the list will
95 not be imported, and their top-level code will not be run as a result.
97 A string or callable alone can also be passed as ``ignore`` without a
98 surrounding list.
100 .. versionadded:: 1.0a3
101 the ``ignore`` argument
102 """
104 pkg_name = package.__name__
106 if ignore is not None and (
107 isinstance(ignore, str) or not hasattr(ignore, "__iter__")
108 ):
109 ignore = [ignore]
110 elif ignore is None:
111 ignore = []
113 # non-leading-dotted name absolute object name
114 str_ignores = [ign for ign in ignore if isinstance(ign, str)]
115 # leading dotted name relative to scanned package
116 rel_ignores = [ign for ign in str_ignores if ign.startswith(".")]
117 # non-leading dotted names
118 abs_ignores = [ign for ign in str_ignores if not ign.startswith(".")]
119 # functions, e.g. re.compile('pattern').search
120 callable_ignores = [ign for ign in ignore if callable(ign)]
122 def _ignore(fullname):
123 for ign in rel_ignores:
124 if fullname.startswith(pkg_name + ign):
125 return True
126 for ign in abs_ignores:
127 # non-leading-dotted name absolute object name
128 if fullname.startswith(ign):
129 return True
130 for ign in callable_ignores:
131 if ign(fullname):
132 return True
133 return False
135 def invoke(mod_name, name, ob):
137 fullname = mod_name + "." + name
139 if _ignore(fullname):
140 return
142 category_keys = categories
143 try:
144 # Some metaclasses do insane things when asked for an
145 # ``ATTACH_ATTR``, like not raising an AttributeError but
146 # some other arbitary exception. Some even shittier
147 # introspected code lets us access ``ATTACH_ATTR`` far but
148 # barfs on a second attribute access for ``attached_to``
149 # (still not raising an AttributeError, but some other
150 # arbitrary exception). Finally, the shittiest code of all
151 # allows the attribute access of the ``ATTACH_ATTR`` *and*
152 # ``attached_to``, (say, both ``ob.__getattr__`` and
153 # ``attached_categories.__getattr__`` returning a proxy for
154 # any attribute access), which either a) isn't callable or b)
155 # is callable, but, when called, shits its pants in an
156 # potentially arbitrary way (although for b, only TypeError
157 # has been seen in the wild, from PyMongo). Thus the
158 # catchall except: return here, which in any other case would
159 # be high treason.
160 attached_categories = getattr(ob, ATTACH_ATTR)
161 if not attached_categories.attached_to(mod_name, name, ob):
162 return
163 except:
164 return
165 if category_keys is None:
166 category_keys = list(attached_categories.keys())
167 try:
168 # When metaclasses return proxies for any attribute access
169 # the list may contain keys of different types which might
170 # not be sortable. In that case we can just return,
171 # because we're not dealing with a proper venusian
172 # callback.
173 category_keys.sort()
174 except TypeError: # pragma: no cover
175 return
176 for category in category_keys:
177 callbacks = attached_categories.get(category, [])
178 try:
179 # Metaclasses might trick us by reaching this far and then
180 # fail with too little values to unpack.
181 for callback, cb_mod_name, liftid, scope in callbacks:
182 if cb_mod_name != mod_name:
183 # avoid processing objects that were imported into
184 # this module but were not actually defined there
185 continue
186 callback(self, name, ob)
187 except ValueError: # pragma: nocover
188 continue
190 for name, ob in getmembers(package):
191 # whether it's a module or a package, we need to scan its
192 # members; walk_packages only iterates over submodules and
193 # subpackages
194 invoke(pkg_name, name, ob)
196 if hasattr(package, "__path__"): # package, not module
197 results = walk_packages(
198 package.__path__,
199 package.__name__ + ".",
200 onerror=onerror,
201 ignore=_ignore,
202 )
204 for importer, modname, ispkg in results:
205 loader = importer.find_module(modname)
206 if loader is not None: # happens on pypy with orphaned pyc
207 try:
208 get_filename = getattr(loader, "get_filename", None)
209 if get_filename is None: # pragma: nocover
210 get_filename = loader._get_filename
211 try:
212 fn = get_filename(modname)
213 except TypeError: # pragma: nocover
214 fn = get_filename()
216 # NB: use __import__(modname) rather than
217 # loader.load_module(modname) to prevent
218 # inappropriate double-execution of module code
219 try:
220 __import__(modname)
221 except Exception:
222 if onerror is not None:
223 onerror(modname)
224 else:
225 raise
226 module = sys.modules.get(modname)
227 if module is not None:
228 for name, ob in getmembers(module, None):
229 invoke(modname, name, ob)
230 finally:
231 if hasattr(loader, "file") and hasattr(
232 loader.file, "close"
233 ): # pragma: nocover
234 loader.file.close()
237class AttachInfo(object):
238 """
239 An instance of this class is returned by the
240 :func:`venusian.attach` function. It has the following
241 attributes:
243 ``scope``
245 One of ``exec``, ``module``, ``class``, ``function call`` or
246 ``unknown`` (each a string). This is the scope detected while
247 executing the decorator which runs the attach function.
249 ``module``
251 The module in which the decorated function was defined.
253 ``locals``
255 A dictionary containing decorator frame's f_locals.
257 ``globals``
259 A dictionary containing decorator frame's f_globals.
261 ``category``
263 The ``category`` argument passed to ``attach`` (or ``None``, the
264 default).
266 ``codeinfo``
268 A tuple in the form ``(filename, lineno, function, sourceline)``
269 representing the context of the venusian decorator used. Eg.
270 ``('/home/chrism/projects/venusian/tests/test_advice.py', 81,
271 'testCallInfo', 'add_handler(foo, bar)')``
273 """
275 def __init__(self, **kw):
276 self.__dict__.update(kw)
279class Categories(dict):
280 def __init__(self, attached_to):
281 super(dict, self).__init__()
282 if isinstance(attached_to, tuple):
283 self.attached_id = attached_to
284 else:
285 self.attached_id = id(attached_to)
286 self.lifted = False
288 def attached_to(self, mod_name, name, obj):
289 if isinstance(self.attached_id, int):
290 return self.attached_id == id(obj)
291 return self.attached_id == (mod_name, name)
294def attach(wrapped, callback, category=None, depth=1, name=None):
295 """ Attach a callback to the wrapped object. It will be found
296 later during a scan. This function returns an instance of the
297 :class:`venusian.AttachInfo` class.
299 ``category`` should be ``None`` or a string representing a decorator
300 category name.
302 ``name`` should be ``None`` or a string representing a subcategory within
303 the category. This will be used by the ``lift`` class decorator to
304 determine if decorations of a method should be inherited or overridden.
305 """
307 frame = sys._getframe(depth + 1)
308 scope, module, f_locals, f_globals, codeinfo = getFrameInfo(frame)
309 module_name = getattr(module, "__name__", None)
310 wrapped_name = getattr(wrapped, "__name__", None)
311 class_name = codeinfo[2]
313 liftid = "%s %s" % (wrapped_name, name)
315 if scope == "class":
316 # we're in the midst of a class statement
317 categories = f_locals.get(ATTACH_ATTR, None)
318 if categories is None or not categories.attached_to(
319 module_name, class_name, None
320 ):
321 categories = Categories((module_name, class_name))
322 f_locals[ATTACH_ATTR] = categories
323 callbacks = categories.setdefault(category, [])
324 else:
325 categories = getattr(wrapped, ATTACH_ATTR, None)
326 if categories is None or not categories.attached_to(
327 module_name, wrapped_name, wrapped
328 ):
329 # if there aren't any attached categories, or we've retrieved
330 # some by inheritance, we need to create new ones
331 categories = Categories(wrapped)
332 setattr(wrapped, ATTACH_ATTR, categories)
333 callbacks = categories.setdefault(category, [])
335 callbacks.append((callback, module_name, liftid, scope))
337 return AttachInfo(
338 scope=scope,
339 module=module,
340 locals=f_locals,
341 globals=f_globals,
342 category=category,
343 codeinfo=codeinfo,
344 )
347def walk_packages(path=None, prefix="", onerror=None, ignore=None):
348 """Yields (module_loader, name, ispkg) for all modules recursively
349 on path, or, if path is None, all accessible modules.
351 'path' should be either None or a list of paths to look for
352 modules in.
354 'prefix' is a string to output on the front of every module name
355 on output.
357 Note that this function must import all *packages* (NOT all
358 modules!) on the given path, in order to access the __path__
359 attribute to find submodules.
361 'onerror' is a function which gets called with one argument (the name of
362 the package which was being imported) if any exception occurs while
363 trying to import a package. If no onerror function is supplied, any
364 exception is exceptions propagated, terminating the search.
366 'ignore' is a function fed a fullly dotted name; if it returns True, the
367 object is skipped and not returned in results (and if it's a package it's
368 not imported).
370 Examples:
372 # list all modules python can access
373 walk_packages()
375 # list all submodules of ctypes
376 walk_packages(ctypes.__path__, ctypes.__name__+'.')
378 # NB: we can't just use pkgutils.walk_packages because we need to ignore
379 # things
380 """
382 def seen(p, m={}):
383 if p in m: # pragma: no cover
384 return True
385 m[p] = True
387 # iter_modules is nonrecursive
388 for importer, name, ispkg in iter_modules(path, prefix):
390 if ignore is not None and ignore(name):
391 # if name is a package, ignoring here will cause
392 # all subpackages and submodules to be ignored too
393 continue
395 # do any onerror handling before yielding
397 if ispkg:
398 try:
399 __import__(name)
400 except Exception:
401 if onerror is not None:
402 onerror(name)
403 else:
404 raise
405 else:
406 yield importer, name, ispkg
407 path = getattr(sys.modules[name], "__path__", None) or []
409 # don't traverse path items we've seen before
410 path = [p for p in path if not seen(p)]
412 for item in walk_packages(path, name + ".", onerror, ignore):
413 yield item
414 else:
415 yield importer, name, ispkg
418class lift(object):
419 """
420 A class decorator which 'lifts' superclass venusian configuration
421 decorations into subclasses. For example::
423 from venusian import lift
424 from somepackage import venusian_decorator
426 class Super(object):
427 @venusian_decorator()
428 def boo(self): pass
430 @venusian_decorator()
431 def hiss(self): pass
433 @venusian_decorator()
434 def jump(self): pass
436 @lift()
437 class Sub(Super):
438 def boo(self): pass
440 def hiss(self): pass
442 @venusian_decorator()
443 def smack(self): pass
445 The above configuration will cause the callbacks of seven venusian
446 decorators. The ones attached to Super.boo, Super.hiss, and Super.jump
447 *plus* ones attached to Sub.boo, Sub.hiss, Sub.hump and Sub.smack.
449 If a subclass overrides a decorator on a method, its superclass decorators
450 will be ignored for the subclass. That means that in this configuration::
452 from venusian import lift
453 from somepackage import venusian_decorator
455 class Super(object):
456 @venusian_decorator()
457 def boo(self): pass
459 @venusian_decorator()
460 def hiss(self): pass
462 @lift()
463 class Sub(Super):
465 def boo(self): pass
467 @venusian_decorator()
468 def hiss(self): pass
470 Only four, not five decorator callbacks will be run: the ones attached to
471 Super.boo and Super.hiss, the inherited one of Sub.boo and the
472 non-inherited one of Sub.hiss. The inherited decorator on Super.hiss will
473 be ignored for the subclass.
475 The ``lift`` decorator takes a single argument named 'categories'. If
476 supplied, it should be a tuple of category names. Only decorators
477 in this category will be lifted if it is suppled.
479 """
481 def __init__(self, categories=None):
482 self.categories = categories
484 def __call__(self, wrapped):
485 if not isclass(wrapped):
486 raise RuntimeError(
487 '"lift" only works as a class decorator; you tried to use '
488 "it against %r" % wrapped
489 )
490 frame = sys._getframe(1)
491 scope, module, f_locals, f_globals, codeinfo = getFrameInfo(frame)
492 module_name = getattr(module, "__name__", None)
493 newcategories = Categories(wrapped)
494 newcategories.lifted = True
495 for cls in getmro(wrapped):
496 attached_categories = cls.__dict__.get(ATTACH_ATTR, None)
497 if attached_categories is None:
498 attached_categories = cls.__dict__.get(LIFTONLY_ATTR, None)
499 if attached_categories is not None:
500 for cname, category in attached_categories.items():
501 if cls is not wrapped:
502 if self.categories and not cname in self.categories:
503 continue
504 callbacks = newcategories.get(cname, [])
505 newcallbacks = []
506 for cb, _, liftid, cscope in category:
507 append = True
508 toappend = (cb, module_name, liftid, cscope)
509 if cscope == "class":
510 for ncb, _, nliftid, nscope in callbacks:
511 if nscope == "class" and liftid == nliftid:
512 append = False
513 if append:
514 newcallbacks.append(toappend)
515 newcategory = list(callbacks) + newcallbacks
516 newcategories[cname] = newcategory
517 if attached_categories.lifted:
518 break
519 if newcategories: # if it has any keys
520 setattr(wrapped, ATTACH_ATTR, newcategories)
521 return wrapped
524class onlyliftedfrom(object):
525 """
526 A class decorator which marks a class as 'only lifted from'. Decorations
527 made on methods of the class won't have their callbacks called directly,
528 but classes which inherit from only-lifted-from classes which also use the
529 ``lift`` class decorator will use the superclass decoration callbacks.
531 For example::
533 from venusian import lift, onlyliftedfrom
534 from somepackage import venusian_decorator
536 @onlyliftedfrom()
537 class Super(object):
538 @venusian_decorator()
539 def boo(self): pass
541 @venusian_decorator()
542 def hiss(self): pass
544 @lift()
545 class Sub(Super):
547 def boo(self): pass
549 def hiss(self): pass
551 Only two decorator callbacks will be run: the ones attached to Sub.boo and
552 Sub.hiss. The inherited decorators on Super.boo and Super.hiss will be
553 not be registered.
554 """
556 def __call__(self, wrapped):
557 if not isclass(wrapped):
558 raise RuntimeError(
559 '"onlyliftedfrom" only works as a class decorator; you tried '
560 "to use it against %r" % wrapped
561 )
562 cats = getattr(wrapped, ATTACH_ATTR, None)
563 class_name = wrapped.__name__
564 module_name = wrapped.__module__
565 key = (module_name, class_name, wrapped)
566 if cats is None or not cats.attached_to(*key):
567 # we either have no categories or our categories are defined
568 # in a superclass
569 return
570 delattr(wrapped, ATTACH_ATTR)
571 setattr(wrapped, LIFTONLY_ATTR, cats)
572 return wrapped