diff --git a/python/helpers/pockets/__init__.py b/python/helpers/pockets/__init__.py index c51701118b2f..4d8d678bc79c 100644 --- a/python/helpers/pockets/__init__.py +++ b/python/helpers/pockets/__init__.py @@ -1,38 +1,28 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2016 the Pockets team, see AUTHORS. +# Copyright (c) 2018 the Pockets team, see AUTHORS. # Licensed under the BSD License, see LICENSE for details. -"""*Let me check my pockets...* +""" +*Let me check my pockets...* Functions available in the `pockets.*` submodules are also imported to the base package for easy access, so:: - from pockets import camel, peek_iter, resolve + from pockets import camel, iterpeek, resolve works just as well as:: from pockets.inspect import resolve - from pockets.iterators import peek_iter + from pockets.iterators import iterpeek from pockets.string import camel """ -from __future__ import absolute_import -from pockets._version import __version__ -from pockets.collections import is_listy, listify, mappify -from pockets.inspect import resolve -from pockets.iterators import peek_iter, modify_iter -from pockets.string import camel, uncamel, splitcaps, UnicodeMixin +from __future__ import absolute_import, print_function + +import sys + +from pockets.inspect import hoist_submodules -__all__ = ["__version__", - "camel", - "uncamel", - "splitcaps", - "UnicodeMixin", - "resolve", - "is_listy", - "listify", - "mappify", - "peek_iter", - "modify_iter"] +hoist_submodules(sys.modules[__name__]) diff --git a/python/helpers/pockets/_version.py b/python/helpers/pockets/_version.py index c099eb584420..40f8e2596bad 100644 --- a/python/helpers/pockets/_version.py +++ b/python/helpers/pockets/_version.py @@ -1,8 +1,10 @@ # Package versioning solution originally found here: # http://stackoverflow.com/q/458550 +__all__ = ["__version__"] + # Store the version here so: # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module -__version__ = '0.3.2' +__version__ = "0.9.1" diff --git a/python/helpers/pockets/autolog.py b/python/helpers/pockets/autolog.py new file mode 100755 index 000000000000..bb200659e854 --- /dev/null +++ b/python/helpers/pockets/autolog.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 the Pockets team, see AUTHORS. +# Licensed under the BSD License, see LICENSE for details. + +""" +An easy to import AutoLogger instance! + +>>> import logging, sys +>>> logging.basicConfig(format="%(name)s: %(message)s", stream=sys.stdout) +>>> from pockets.autolog import log +>>> log.error("Always log from the correct module.Class!") # doctest: +SKIP +pockets.autolog: Always log from the correct module.Class! + +See Also: + `pockets.logger.AutoLogger` +""" + +from __future__ import absolute_import, print_function + +from pockets.logging import AutoLogger + + +__all__ = ["log"] + + +log = AutoLogger() diff --git a/python/helpers/pockets/collections.py b/python/helpers/pockets/collections.py index 144ee82790a7..dbc044b1f009 100644 --- a/python/helpers/pockets/collections.py +++ b/python/helpers/pockets/collections.py @@ -1,32 +1,230 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2016 the Pockets team, see AUTHORS. +# Copyright (c) 2018 the Pockets team, see AUTHORS. # Licensed under the BSD License, see LICENSE for details. -"""A pocket full of useful collection functions!""" +"""A pocket full of useful collection tools!""" -from __future__ import absolute_import -from collections import Sized, Iterable, Mapping +from __future__ import absolute_import, print_function + +from collections import defaultdict from inspect import isclass +try: + from collections.abc import Iterable, Mapping, Sized +except ImportError: + from collections import Iterable, Mapping, Sized +try: + from collections import OrderedDict +except ImportError: + OrderedDict = dict + import six -__all__ = ["is_listy", "listify", "mappify"] + +__all__ = [ + "groupify", + "keydefaultdict", + "is_listy", + "listify", + "is_mappy", + "mappify", + "nesteddefaultdict", + "readable_join", + "uniquify", +] + + +def groupify(items, keys, val_key=None): + """ + Groups a list of items into nested OrderedDicts based on the given keys. + + Note: + On Python 2.6 the return value will use regular dicts instead of + OrderedDicts. + + >>> from __future__ import print_function + >>> from json import dumps + >>> + >>> ex = lambda x: print(dumps(x, indent=2, sort_keys=True, default=repr)) + >>> + >>> class Reminder: + ... def __init__(self, when, where, what): + ... self.when = when + ... self.where = where + ... self.what = what + ... def __repr__(self): + ... return 'Reminder({0.when}, {0.where}, {0.what})'.format(self) + ... + >>> reminders = [ + ... Reminder('Fri', 'Home', 'Eat cereal'), + ... Reminder('Fri', 'Work', 'Feed Ivan'), + ... Reminder('Sat', 'Home', 'Sleep in'), + ... Reminder('Sat', 'Home', 'Play Zelda'), + ... Reminder('Sun', 'Home', 'Sleep in'), + ... Reminder('Sun', 'Work', 'Reset database')] + >>> + >>> ex(groupify(reminders, 'when')) + { + "Fri": [ + "Reminder(Fri, Home, Eat cereal)", + "Reminder(Fri, Work, Feed Ivan)" + ], + "Sat": [ + "Reminder(Sat, Home, Sleep in)", + "Reminder(Sat, Home, Play Zelda)" + ], + "Sun": [ + "Reminder(Sun, Home, Sleep in)", + "Reminder(Sun, Work, Reset database)" + ] + } + >>> + >>> ex(groupify(reminders, ['when', 'where'])) + { + "Fri": { + "Home": [ + "Reminder(Fri, Home, Eat cereal)" + ], + "Work": [ + "Reminder(Fri, Work, Feed Ivan)" + ] + }, + "Sat": { + "Home": [ + "Reminder(Sat, Home, Sleep in)", + "Reminder(Sat, Home, Play Zelda)" + ] + }, + "Sun": { + "Home": [ + "Reminder(Sun, Home, Sleep in)" + ], + "Work": [ + "Reminder(Sun, Work, Reset database)" + ] + } + } + >>> + >>> ex(groupify(reminders, ['when', 'where'], 'what')) + { + "Fri": { + "Home": [ + "Eat cereal" + ], + "Work": [ + "Feed Ivan" + ] + }, + "Sat": { + "Home": [ + "Sleep in", + "Play Zelda" + ] + }, + "Sun": { + "Home": [ + "Sleep in" + ], + "Work": [ + "Reset database" + ] + } + } + >>> + >>> ex(groupify(reminders, lambda r: '{0.when} - {0.where}'.format(r), 'what')) + { + "Fri - Home": [ + "Eat cereal" + ], + "Fri - Work": [ + "Feed Ivan" + ], + "Sat - Home": [ + "Sleep in", + "Play Zelda" + ], + "Sun - Home": [ + "Sleep in" + ], + "Sun - Work": [ + "Reset database" + ] + } + + Args: + items (list): The list of items to arrange in groups. + keys (str|callable|list): The key or keys that should be used to group + `items`. If multiple keys are given, then each will correspond to + an additional level of nesting in the order they are given. + val_key (str|callable): A key or callable used to generate the leaf + values in the nested OrderedDicts. If `val_key` is `None`, then + the item itself is used. Defaults to `None`. + + Returns: + OrderedDict: Nested OrderedDicts with `items` grouped by `keys`. + + """ # noqa: E501 + + if not keys: + return items + keys = listify(keys) + last_key = keys[-1] + is_callable = callable(val_key) + groupified = OrderedDict() + for item in items: + current = groupified + for key in keys: + attr = key(item) if callable(key) else getattr(item, key) + if attr not in current: + current[attr] = [] if key is last_key else OrderedDict() + current = current[attr] + if val_key: + value = val_key(item) if is_callable else getattr(item, val_key) + else: + value = item + current.append(value) + return groupified + + +class keydefaultdict(defaultdict): + """ + A defaultdict that passes the missed key to the factory function. + + >>> def echo_factory(missing_key): + ... return missing_key + ... + >>> d = keydefaultdict(echo_factory) + >>> d['Hello World'] + 'Hello World' + >>> d['Hello World'] = 'Goodbye' + >>> d['Hello World'] + 'Goodbye' + + """ + + def __missing__(self, key): + if self.default_factory is None: + raise KeyError(key) + else: + ret = self[key] = self.default_factory(key) + return ret def is_listy(x): - """Return True if `x` is "listy", i.e. a list-like object. + """ + Return True if `x` is "listy", i.e. a list-like object. "Listy" is defined as a sized iterable which is neither a map nor a string: - >>> is_listy(["a", "b"]) + >>> is_listy(['a', 'b']) True >>> is_listy(set()) True - >>> is_listy(iter(["a", "b"])) + >>> is_listy(iter(['a', 'b'])) False - >>> is_listy({"a": "b"}) + >>> is_listy({'a': 'b'}) False - >>> is_listy("a regular string") + >>> is_listy('a regular string') False Note: @@ -40,24 +238,30 @@ def is_listy(x): bool: True if `x` is "listy", False otherwise. """ - return (isinstance(x, Sized) and - isinstance(x, Iterable) and - not isinstance(x, Mapping) and - not isinstance(x, six.string_types)) + return ( + isinstance(x, Sized) + and isinstance(x, Iterable) + and not isinstance(x, (Mapping, type(b""))) + and not isinstance(x, six.string_types) + ) def listify(x, minlen=0, default=None, cls=None): - """Return a listified version of `x`. + """ + Return a listified version of `x`. If `x` is a non-string iterable, it is wrapped in a list; otherwise - a list is returned with `x` as its only element. + a list is returned with `x` as its only element. If `x` is `None`, an + empty list is returned. - >>> listify("a regular string") + >>> listify('a regular string') ['a regular string'] - >>> listify(tuple(["a", "b", "c"])) + >>> listify(tuple(['a', 'b', 'c'])) ['a', 'b', 'c'] >>> listify({'a': 'A'}) [{'a': 'A'}] + >>> listify(None) + [] Note: Not guaranteed to return a copy of `x`. If `x` is already a list and @@ -74,15 +278,15 @@ def listify(x, minlen=0, default=None, cls=None): [] >>> listify([], minlen=1) [None] - >>> listify("item", minlen=3) + >>> listify('item', minlen=3) ['item', None, None] default (any value): Value that should be used to pad the list if it would be shorter than `minlen`: - >>> listify([], minlen=1, default="PADDING") + >>> listify([], minlen=1, default='PADDING') ['PADDING'] - >>> listify("item", minlen=3, default="PADDING") + >>> listify('item', minlen=3, default='PADDING') ['item', 'PADDING', 'PADDING'] cls (class or callable): Instead of wrapping `x` in a list, wrap it @@ -90,7 +294,7 @@ def listify(x, minlen=0, default=None, cls=None): as its single parameter when called: >>> from collections import deque - >>> listify(["a", "b", "c"], cls=deque) + >>> listify(['a', 'b', 'c'], cls=deque) deque(['a', 'b', 'c']) Returns: @@ -110,21 +314,56 @@ def listify(x, minlen=0, default=None, cls=None): return x +def is_mappy(x): + """ + Return True if `x` is "mappy", i.e. a map-like object. + + "Mappy" is defined as any instance of `collections.Mapping`: + + >>> is_mappy({'a': 'b'}) + True + >>> from collections import defaultdict + >>> is_mappy(defaultdict(list)) + True + >>> is_mappy('a regular string') + False + >>> is_mappy(['a', 'b']) + False + >>> is_listy(iter({'a': 'b'})) + False + + Note: + Iterables and generators fail the "mappy" test. + + Args: + x (any value): The object to test. + + Returns: + bool: True if `x` is "mappy", False otherwise. + + """ + return isinstance(x, Mapping) + + def mappify(x, default=True, cls=None): - """Return a mappified version of `x`. + """ + Return a mappified version of `x`. If `x` is a string, it becomes the only key of the returned dict. If `x` is a non-string iterable, the elements of `x` become keys in the returned - dict. The values of the returned dict are set to `default`. + dict. The values of the returned dict are set to `default`. If `x` is + `None`, an empty dict is returned. If `x` is a map, it is returned directly. - >>> mappify("a regular string") + >>> mappify('a regular string') {'a regular string': True} - >>> mappify(["a"]) + >>> mappify(['a']) {'a': True} - >>> mappify({'a': "A"}) + >>> mappify({'a': 'A'}) {'a': 'A'} + >>> mappify(None) + {} Note: Not guaranteed to return a copy of `x`. If `x` is already a map and @@ -141,7 +380,7 @@ def mappify(x, default=True, cls=None): its single parameter when called: >>> from collections import defaultdict - >>> mappify("a", cls=lambda x: defaultdict(None, x)) + >>> mappify('a', cls=lambda x: defaultdict(None, x)) defaultdict(None, {'a': True}) Returns: @@ -151,13 +390,111 @@ def mappify(x, default=True, cls=None): TypeError: If `x` is not a map, iterable, or string. """ - if not isinstance(x, Mapping): + if x is None: + x = {} + elif not isinstance(x, Mapping): if isinstance(x, six.string_types): x = {x: default} elif isinstance(x, Iterable): - x = dict([(v, default) for v in x]) + # If cls is specified, attempt to preserve the order of x, in + # case cls is also a class that preserves order. + arg = [(v, default) for v in x] + x = OrderedDict(arg) if cls else dict(arg) else: - raise TypeError("Unable to mappify {0}".format(type(x)), x) + raise TypeError( + "Unable to mappify non-mappy {0}".format(type(x)), x + ) + + if cls and not (isclass(cls) and issubclass(type(x), cls)): + x = cls(x) + return x + + +def nesteddefaultdict(): + """ + A defaultdict that returns nested defaultdicts as the default value. + + Each defaultdict returned as the default value will also return nested + defaultdicts, and so on. + + >>> nested = nesteddefaultdict() + >>> nested_child = nested['New Key 1'] + >>> nested_child + defaultdict(...) + >>> nested_grandchild = nested_child['New Key 2'] + >>> nested_grandchild + defaultdict(...) + + """ + return defaultdict(nesteddefaultdict) + + +def readable_join(xs, conjunction="and", sep=","): + """ + Accepts a list of strings and separates them with commas as grammatically + appropriate with a conjunction before the final entry. Any input strings + containing only whitespace will not be included in the result. + + >>> readable_join(['foo']) + 'foo' + >>> readable_join(['foo', 'bar']) + 'foo and bar' + >>> readable_join(['foo', 'bar', 'baz']) + 'foo, bar, and baz' + >>> readable_join(['foo', ' ', '', 'bar', '', ' ', 'baz']) + 'foo, bar, and baz' + >>> readable_join(['foo', 'bar', 'baz'], 'or') + 'foo, bar, or baz' + >>> readable_join(['foo', 'bar', 'baz'], 'but never') + 'foo, bar, but never baz' + + """ + xs = [s for s in map(lambda s: str(s).strip(), listify(xs)) if s] + if len(xs) > 1: + xs = list(xs) + xs[-1] = conjunction + " " + xs[-1] + return (sep + " " if len(xs) > 2 else " ").join(xs) + + +def uniquify(x, key=lambda o: o, cls=None): + """ + Returns an order-preserved copy of `x` with duplicate items removed. + + >>> uniquify(['a', 'z', 'a', 'b', 'a', 'y', 'a', 'c', 'a', 'x']) + ['a', 'z', 'b', 'y', 'c', 'x'] + + Args: + x (Sequence): Sequence to uniquify. + + key (str or callable): Similar to `sorted`, specifies an attribute or + function of one argument that is used to extract a comparison key + from each list element: key=str.lower. By default, compares the + elements directly. + + >>> strings = ['ASDF', 'asdf', 'ZXCV', 'zxcv'] + >>> uniquify(strings, key=str.lower) + ['ASDF', 'ZXCV'] + + cls (class or callable): Instead of wrapping `x` in a list, wrap it + in an instance of `cls`. `cls` should accept an iterable object + as its single parameter when called: + + >>> from collections import deque + >>> listify(['a', 'b', 'c'], cls=deque) + deque(['a', 'b', 'c']) + + Returns: + list: An order-preserved copy of `x` with duplicate items removed. + + Raises: + TypeError: If `x` is not "listy". + + """ + if not is_listy(x): + raise TypeError("Unable to uniquify non-listy {0}".format(type(x)), x) + seen = set() + keys = [(key(o) if callable(key) else getattr(o, key), o) for o in x] + x = [o for k, o in keys if k not in seen and not seen.add(k)] if cls and not (isclass(cls) and issubclass(type(x), cls)): x = cls(x) diff --git a/python/helpers/pockets/datetime.py b/python/helpers/pockets/datetime.py new file mode 100644 index 000000000000..dafff82a36ad --- /dev/null +++ b/python/helpers/pockets/datetime.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 the Pockets team, see AUTHORS. +# Licensed under the BSD License, see LICENSE for details. + +"""A pocket full of useful datetime tools!""" + +from __future__ import absolute_import, print_function + +import sys +from datetime import datetime, timedelta + + +__all__ = [ + "ceil_datetime", + "floor_datetime", + "round_datetime", + "timedelta_total_seconds", +] + + +if sys.version_info < (2, 7): + + def timedelta_total_seconds(td): + """ + Python 2.6 replacement function for timedelta.total_seconds(). + + Args: + td (datetime.timedelta): A `datetime.timedelta` instance. + + Returns: + float: The total number of seconds plus the fractional + number of microseconds in `td`. + + """ + total_seconds = td.seconds + (td.days * 24.0 * 3600.0) + return (td.microseconds + (total_seconds * 1.0e6)) / 1.0e6 + + +else: + + def timedelta_total_seconds(td): + """ + Python 2.6 replacement function for timedelta.total_seconds(). + + Args: + td (datetime.timedelta): A `datetime.timedelta` instance. + + Returns: + float: The total number of seconds plus the fractional + number of microseconds in `td`. + + """ + return td.total_seconds() + + +def ceil_datetime(dt, nearest): + """ + Rounds the given `datetime` up to the nearest `timedelta` increment. + + Note: + `dt.microsecond` is always set to zero and ignored. + + Args: + dt (datetime.datetime): The `datetime` instance to ceil. + nearest (datetime.timedelta): The `timedelta` to use as the increment. + + Returns: + datetime.datetime: A copy of `dt` ceiled to `nearest`. + + """ + dt = dt.replace(microsecond=0) + dt_min = datetime.min.replace(tzinfo=dt.tzinfo) + secs = timedelta_total_seconds(dt_min - dt) + nearest_secs = timedelta_total_seconds(nearest) + total_delta_secs = secs % nearest_secs + delta_days = total_delta_secs // 86400 # (60 * 60 * 24) + delta_secs = total_delta_secs % 86400.0 # (60 * 60 * 24) + return dt + timedelta(days=delta_days, seconds=delta_secs) + + +def floor_datetime(dt, nearest): + """ + Rounds the given `datetime` down to the nearest `timedelta` increment. + + Note: + `dt.microsecond` is always set to zero and ignored. + + Args: + dt (datetime.datetime): The `datetime` instance to floor. + nearest (datetime.timedelta): The `timedelta` to use as the increment. + + Returns: + datetime.datetime: A copy of `dt` floored to `nearest`. + + """ + dt = dt.replace(microsecond=0) + dt_min = datetime.min.replace(tzinfo=dt.tzinfo) + secs = timedelta_total_seconds(dt - dt_min) + nearest_secs = timedelta_total_seconds(nearest) + total_delta_secs = secs % nearest_secs + delta_days = total_delta_secs // 86400 # (60 * 60 * 24) + delta_secs = total_delta_secs % 86400.0 # (60 * 60 * 24) + return dt - timedelta(days=delta_days, seconds=delta_secs) + + +def round_datetime(dt, nearest): + """ + Rounds the given `datetime` up/down to the nearest `timedelta` increment. + + Note: + `dt.microsecond` is always set to zero and ignored. + + Args: + dt (datetime.datetime): The `datetime` instance to round. + nearest (datetime.timedelta): The `timedelta` to use as the increment. + + Returns: + datetime.datetime: A copy of `dt` rounded to `nearest`. + + """ + dt = dt.replace(microsecond=0) + dt_min = datetime.min.replace(tzinfo=dt.tzinfo) + secs = timedelta_total_seconds(dt - dt_min) + nearest_secs = timedelta_total_seconds(nearest) + rounded_secs = ((secs + (nearest_secs / 2)) // nearest_secs) * nearest_secs + total_delta_secs = rounded_secs - secs + delta_days = total_delta_secs // 86400 # (60 * 60 * 24) + delta_secs = total_delta_secs % 86400.0 # (60 * 60 * 24) + return dt + timedelta(days=delta_days, seconds=delta_secs) diff --git a/python/helpers/pockets/decorators.py b/python/helpers/pockets/decorators.py new file mode 100644 index 000000000000..5b919091248e --- /dev/null +++ b/python/helpers/pockets/decorators.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 the Pockets team, see AUTHORS. +# Licensed under the BSD License, see LICENSE for details. + +"""A pocket full of useful decorators!""" + +from __future__ import absolute_import, print_function + +import inspect +from functools import wraps + +from pockets.collections import listify +from pockets.inspect import unwrap + + +__all__ = [ + "argmod", + "cached_classproperty", + "cached_property", + "classproperty", +] + + +def argmod(*args): + """ + Decorator that intercepts and modifies function arguments. + + Args: + from_param (str|list): A parameter or list of possible parameters that + should be modified using `modifier_func`. Passing a list of + possible parameters is useful when a function's parameter names + have changed, but you still want to support the old parameter + names. + to_param (str): Optional. If given, to_param will be used as the + parameter name for the modified argument. If not given, to_param + will default to the last parameter given in `from_param`. + modifier_func (callable): The function used to modify the `from_param`. + + Returns: + function: A function that modifies the given `from_param` before the + function is called. + """ + from_param = listify(args[0]) + to_param = from_param[-1] if len(args) < 3 else args[1] + modifier_func = args[-1] + + def _decorator(func): + try: + argspec = inspect.getfullargspec(unwrap(func)) + except AttributeError: + argspec = inspect.getargspec(unwrap(func)) + if to_param not in argspec.args: + return func + arg_index = argspec.args.index(to_param) + + @wraps(func) + def _modifier(*args, **kwargs): + kwarg = False + for arg in from_param: + if arg in kwargs: + kwarg = arg + break + + if kwarg: + kwargs[to_param] = modifier_func(kwargs.pop(kwarg)) + elif arg_index < len(args): + args = list(args) + args[arg_index] = modifier_func(args[arg_index]) + return func(*args, **kwargs) + + return _modifier + + return _decorator + + +class cached_classproperty(property): + """ + Like @cached_property except it works on classes instead of instances. + + + Note: + Class properties created by @cached_classproperty are read-only. + Any attempts to write to the property will erase the + @cached_classproperty, and the behavior of the underlying method + will be lost. + + >>> class MyClass(object): + ... @cached_classproperty + ... def myproperty(cls): + ... return '{0}.myproperty'.format(cls.__name__) + >>> MyClass.myproperty + 'MyClass.myproperty' + + """ + + def __init__(self, fget, *arg, **kw): + super(cached_classproperty, self).__init__(fget, *arg, **kw) + self.__doc__ = fget.__doc__ + self.__fget_name__ = fget.__name__ + + def __get__(desc, self, cls): + cache_attr = "_cached_{0}_{1}".format(cls.__name__, desc.__fget_name__) + if not hasattr(cls, cache_attr): + setattr(cls, cache_attr, desc.fget(cls)) + return getattr(cls, cache_attr) + + def getter(self, fget): + raise AttributeError("@cached_classproperty.getter is not supported") + + def setter(self, fset): + raise AttributeError("@cached_classproperty.setter is not supported") + + def deleter(self, fdel): + raise AttributeError("@cached_classproperty.deleter is not supported") + + +def cached_property(func): + """Decorator for making readonly, memoized properties.""" + cache_attr = "_cached_{0}".format(func.__name__) + + @property + @wraps(func) + def caching(self, *args, **kwargs): + if not hasattr(self, cache_attr): + setattr(self, cache_attr, func(self, *args, **kwargs)) + return getattr(self, cache_attr) + + return caching + + +class classproperty(property): + """ + Decorator to create a read-only class property similar to classmethod. + + For whatever reason, the @property decorator isn't smart enough to + recognize @classmethods and behaves differently on them than on instance + methods. This decorator may be used like to create a class-level property, + useful for singletons and other one-per-class properties. + + This implementation is partially based on + `sqlalchemy.util.langhelpers.classproperty`. + + Note: + Class properties created by @classproperty are read-only. Any attempts + to write to the property will erase the @classproperty, and the + behavior of the underlying method will be lost. + + >>> class MyClass(object): + ... @classproperty + ... def myproperty(cls): + ... return '{0}.myproperty'.format(cls.__name__) + >>> MyClass.myproperty + 'MyClass.myproperty' + + """ + + def __init__(self, fget, *arg, **kw): + super(classproperty, self).__init__(fget, *arg, **kw) + self.__doc__ = fget.__doc__ + + def __get__(desc, self, cls): + return desc.fget(cls) + + def getter(self, fget): + raise AttributeError("@classproperty.getter is not supported") + + def setter(self, fset): + raise AttributeError("@classproperty.setter is not supported") + + def deleter(self, fdel): + raise AttributeError("@classproperty.deleter is not supported") diff --git a/python/helpers/pockets/inspect.py b/python/helpers/pockets/inspect.py index 709e3727ece4..37b4966cbaf4 100644 --- a/python/helpers/pockets/inspect.py +++ b/python/helpers/pockets/inspect.py @@ -1,21 +1,245 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2016 the Pockets team, see AUTHORS. +# Copyright (c) 2018 the Pockets team, see AUTHORS. # Licensed under the BSD License, see LICENSE for details. """A pocket full of useful reflection functions!""" -from __future__ import absolute_import +from __future__ import absolute_import, print_function + import inspect import functools +from os.path import basename +from pkgutil import iter_modules -from pockets.collections import listify +import six from six import string_types -__all__ = ["resolve"] +from pockets.collections import listify +from pockets.string import splitify + + +__all__ = [ + "collect_subclasses", + "collect_superclasses", + "collect_superclass_attr_names", + "hoist_submodules", + "import_star", + "import_submodules", + "is_data", + "resolve", + "unwrap", +] + + +def collect_subclasses(cls): + """ + Recursively collects all descendant subclasses that inherit from the + given class, not including the class itself. + + Note: + Does not include `cls` itself. + + Args: + cls (class): The class object from which the collection should begin. + + Returns: + list: A list of `class` objects that inherit from `cls`. This list + will not include `cls` itself. + """ + subclasses = set() + for subclass in cls.__subclasses__(): + subclasses.add(subclass) + subclasses.update(collect_subclasses(subclass)) + return list(subclasses) + + +def collect_superclasses(cls, terminal_class=None, modules=None): + """ + Recursively collects all ancestor superclasses in the inheritance + hierarchy of the given class, including the class itself. + + Note: + Inlcudes `cls` itself. Will not include `terminal_class`. + + Args: + cls (class): The class object from which the collection should begin. + terminal_class (class or list): If `terminal_class` is encountered in + the hierarchy, we stop ascending the tree. `terminal_class` will + not be included in the returned list. + modules (string, module, or list): If `modules` is passed, we only + return classes that are in the given module/modules. This can be + used to exclude base classes that come from external libraries. + + Returns: + list: A list of `class` objects from which `cls` inherits. This list + will include `cls` itself. + """ + terminal_class = listify(terminal_class) + if modules is not None: + modules = listify(modules) + module_strings = [] + for m in modules: + if isinstance(m, six.string_types): + module_strings.append(m) + else: + module_strings.append(m.__name__) + modules = module_strings + + superclasses = set() + is_in_module = modules is None or cls.__module__ in modules + if is_in_module and cls not in terminal_class: + superclasses.add(cls) + for base in cls.__bases__: + superclasses.update( + collect_superclasses(base, terminal_class, modules) + ) + + return list(superclasses) + + +def collect_superclass_attr_names(cls, terminal_class=None, modules=None): + """ + Recursively collects all attribute names of ancestor superclasses in the + inheritance hierarchy of the given class, including the class itself. + + Note: + Inlcudes `cls` itself. Will not include `terminal_class`. + + Args: + cls (class): The class object from which the collection should begin. + terminal_class (class or list): If `terminal_class` is encountered in + the hierarchy, we stop ascending the tree. Attributes from + `terminal_class` will not be included in the returned list. + modules (string, module, or list): If `modules` is passed, we only + return classes that are in the given module/modules. This can be + used to exclude base classes that come from external libraries. + + Returns: + list: A list of `str` attribute names for every `class` in the + inheritance hierarchy. + """ + superclasses = collect_superclasses(cls, terminal_class, modules) + attr_names = set() + for superclass in superclasses: + attr_names.update(superclass.__dict__.keys()) + return list(attr_names) + + +def hoist_submodules(package, extend_all=True): + """ + Sets `__all__` attrs from submodules of `package` as attrs on `package`. + + Note: + This only considers attributes exported by `__all__`. If a submodule + does not define `__all__`, then it is ignored. + + Effectively does:: + + from package.* import * + + Args: + package (str or module): The parent package into which submodule + exports should be hoisted. + extend_all (bool): If True, `package.__all__` will be extended + to include the hoisted attributes. Defaults to True. + + Returns: + list: List of all hoisted attribute names. + + """ + module = resolve(package) + hoisted_attrs = [] + for submodule in import_submodules(module): + for attr_name, attr in import_star(submodule).items(): + hoisted_attrs.append(attr_name) + setattr(module, attr_name, attr) + + if extend_all: + if getattr(module, "__all__", None) is None: + module.__all__ = list(hoisted_attrs) + else: + module.__all__.extend(hoisted_attrs) + + return hoisted_attrs + + +def import_star(module): + """ + Imports all exported attributes of `module` and returns them in a `dict`. + + Note: + This only considers attributes exported by `__all__`. If `module` + does not define `__all__`, then nothing is imported. + + Effectively does:: + + from module import * + + Args: + module (str or module): The module from which a wildcard import + should be done. + + Returns: + dict: Map of all imported attributes. + + """ + module = resolve(module) + attrs = getattr(module, "__all__", []) + return dict([(attr, getattr(module, attr)) for attr in attrs]) + + +def import_submodules(package): + """ + Imports all submodules of `package`. + + Effectively does:: + + __import__(package.*) + + Args: + package (str or module): The parent package from which submodules + should be imported. + + Yields: + module: The next submodule of `package`. + + """ + module = resolve(package) + if basename(module.__file__).startswith("__init__.py"): + for _, submodule_name, _ in iter_modules(module.__path__): + yield resolve(submodule_name, module) + + +def is_data(obj): + """ + Returns True if `obj` is a "data like" object. + + Strongly inspired by `inspect.classify_class_attrs`. This function is + useful when trying to determine if an attribute has a meaningful docstring + or not. In general, a routine can have meaningful docstrings, whereas + non-routines cannot. + + See Also: + * `inspect.classify_class_attrs` + * `inspect.isroutine` + + Args: + obj (object): The object in question. + + Returns: + bool: True if `obj` is "data like", False otherwise. + """ + if isinstance( + obj, (staticmethod, classmethod, property) + ) or inspect.isroutine(obj): + return False + else: + return True def resolve(name, modules=None): - """Resolve a dotted name to an object (usually class, module, or function). + """ + Resolve a dotted name to an object (usually class, module, or function). If `name` is a string, attempt to resolve it according to Python dot notation, e.g. "path.to.MyClass". If `name` is anything other than a @@ -23,7 +247,7 @@ def resolve(name, modules=None): >>> resolve("calendar.TextCalendar") - >>> resolve(object()) #doctest: +ELLIPSIS + >>> resolve(object()) If `modules` is specified, then resolution of `name` is restricted @@ -37,6 +261,8 @@ def resolve(name, modules=None): no leading dots, resolution is first attempted absolutely and then relative to the calling module. + Pass an empty string for `modules` to only use absolute resolution. + Warning: Do not resolve strings supplied by an end user without specifying `modules`. Instantiating an arbitrary object specified by an end user @@ -47,18 +273,18 @@ def resolve(name, modules=None): Restricting `name` resolution to a set of `modules`: - >>> resolve("pockets.camel") #doctest: +ELLIPSIS + >>> resolve("pockets.camel") - >>> resolve("pockets.camel", modules=["re", "six"]) #doctest: +ELLIPSIS + >>> resolve("pockets.camel", modules=["re", "six"]) Traceback (most recent call last): - ... ValueError: Unable to resolve 'pockets.camel' in modules: ['re', 'six'] + ... Args: name (str or object): A dotted name. - modules (str or list, optional): A module or list of modules, under - which to search for `name`. + modules (str, module, or list, optional): A module or list of modules, + under which to search for `name`. Returns: object: The object specified by `name`. @@ -70,16 +296,16 @@ def resolve(name, modules=None): if not isinstance(name, string_types): return name - obj_path = name.split('.') + obj_path = splitify(name, ".", include_empty=True) search_paths = [] - if modules: - while not obj_path[0]: + if modules is not None: + while not obj_path[0].strip(): obj_path.pop(0) for module_path in listify(modules): - search_paths.append(module_path.split('.') + obj_path) + search_paths.append(splitify(module_path, ".") + obj_path) else: caller = inspect.getouterframes(inspect.currentframe())[1][0].f_globals - module_path = caller['__name__'].split('.') + module_path = caller["__name__"].split(".") if not obj_path[0]: obj_path.pop(0) while not obj_path[0]: @@ -93,13 +319,78 @@ def resolve(name, modules=None): search_paths.append(obj_path) search_paths.append(module_path + obj_path) + exceptions = [] for path in search_paths: - try: - obj = functools.reduce(getattr, path[1:], __import__(path[0])) - except (AttributeError, ImportError): - pass - else: - return obj + # Import the most deeply nested module available + module = None + module_path = [] + obj_path = list(path) + while obj_path: + module_name = obj_path.pop(0) + while not module_name: + module_name = obj_path.pop(0) + if isinstance(module_name, string_types): + package = ".".join(module_path + [module_name]) + try: + module = __import__(package, fromlist=module_name) + except ImportError as ex: + exceptions.append(ex) + obj_path = [module_name] + obj_path + break + else: + module_path.append(module_name) + else: + module = module_name + module_path.append(module.__name__) - raise ValueError("Unable to resolve '{0}' " - "in modules: {1}".format(name, modules)) + if module: + if obj_path: + try: + return functools.reduce(getattr, obj_path, module) + except AttributeError as ex: + exceptions.append(ex) + else: + return module + + if modules: + msg = "Unable to resolve '{0}' in modules: {1}".format(name, modules) + else: + msg = "Unable to resolve '{0}'".format(name) + + if exceptions: + msgs = ["{0}: {1}".format(type(e).__name__, e) for e in exceptions] + raise ValueError("\n ".join([msg] + msgs)) + else: + raise ValueError(msg) + + +def unwrap(func): + """ + Finds the innermost function that has been wrapped using `functools.wrap`. + + Note: + This function relies on the existence of the `__wrapped__` attribute, + which was not automatically added until Python 3.2. If you are using + an older version of Python, you'll have to manually add the + `__wrapped__` attribute in order to use `unwrap`:: + + def my_decorator(func): + @wraps(func) + def with_my_decorator(*args, **kwargs): + return func(*args, **kwargs) + + if not hasattr(with_my_decorator, '__wrapped__'): + with_my_decorator.__wrapped__ = func + + return with_my_decorator + + Args: + func (function): A function that may or may not have been wrapped + using `functools.wrap`. + + Returns: + function: The original function before it was wrapped using + `functools.wrap`. `func` is returned directly, if it was never + wrapped using `functools.wrap`. + """ + return unwrap(func.__wrapped__) if hasattr(func, "__wrapped__") else func diff --git a/python/helpers/pockets/iterators.py b/python/helpers/pockets/iterators.py index 5be571e4e187..c93d4957d11a 100644 --- a/python/helpers/pockets/iterators.py +++ b/python/helpers/pockets/iterators.py @@ -1,21 +1,24 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2016 the Pockets team, see AUTHORS. +# Copyright (c) 2018 the Pockets team, see AUTHORS. # Licensed under the BSD License, see LICENSE for details. """A pocket full of useful iterators!""" -from __future__ import absolute_import +from __future__ import absolute_import, print_function + import collections import six -__all__ = ["peek_iter", "modify_iter"] + +__all__ = ["itermod", "iterpeek", "modify_iter", "peek_iter"] -class peek_iter(object): - """An iterator object that supports peeking ahead. +class iterpeek(object): + """ + An iterator object that supports peeking ahead. - >>> p = peek_iter(["a", "b", "c", "d", "e"]) + >>> p = iterpeek(["a", "b", "c", "d", "e"]) >>> p.peek() 'a' >>> p.next() @@ -39,17 +42,18 @@ class peek_iter(object): raised, otherwise the value will be returned. See Also: - `peek_iter` can operate as a drop in replacement for the built-in - `iter `_ + `iterpeek` can operate as a drop in replacement for the built-in + `iter `_ function. Attributes: sentinel (any value): The value used to indicate the iterator is - exhausted. If `sentinel` was not given when the `peek_iter` was + exhausted. If `sentinel` was not given when the `iterpeek` was instantiated, then it will be set to a new object instance: ``object()``. """ + def __init__(self, *args): """__init__(o, sentinel=None)""" self._iterable = iter(*args) @@ -62,7 +66,7 @@ class peek_iter(object): def __next__(self, n=None): # NOTE: Prevent 2to3 from transforming self.next() in next(self), # which causes an infinite loop! - return getattr(self, 'next')(n) + return getattr(self, "next")(n) def _fillcache(self, n): """Cache `n` items. If `n` is 0 or None, then 1 item is cached.""" @@ -76,7 +80,8 @@ class peek_iter(object): self._cache.append(self.sentinel) def has_next(self): - """Determine if iterator is exhausted. + """ + Determine if iterator is exhausted. Returns: bool: True if iterator has more items, False otherwise. @@ -88,7 +93,8 @@ class peek_iter(object): return self.peek() != self.sentinel def next(self, n=None): - """Get the next item or `n` items of the iterator. + """ + Get the next item or `n` items of the iterator. Args: n (int, optional): The number of items to retrieve. Defaults to @@ -100,7 +106,7 @@ class peek_iter(object): the items will be returned in a list. If `n` is 0, an empty list is returned: - >>> p = peek_iter(["a", "b", "c", "d", "e"]) + >>> p = iterpeek(["a", "b", "c", "d", "e"]) >>> p.next() 'a' >>> p.next(0) @@ -130,7 +136,8 @@ class peek_iter(object): return result def peek(self, n=None): - """Preview the next item or `n` items of the iterator. + """ + Preview the next item or `n` items of the iterator. The iterator is not advanced when peek is called. @@ -144,10 +151,10 @@ class peek_iter(object): the items will be returned in a list. If `n` is 0, an empty list is returned. - If the iterator is exhausted, `peek_iter.sentinel` is returned, + If the iterator is exhausted, `iterpeek.sentinel` is returned, or placed as the last item in the returned list: - >>> p = peek_iter(["a", "b", "c"]) + >>> p = iterpeek(["a", "b", "c"]) >>> p.sentinel = "END" >>> p.peek() 'a' @@ -172,8 +179,13 @@ class peek_iter(object): return result -class modify_iter(peek_iter): - """An iterator object that supports modifying items as they are returned. +# Backwards compatibility +peek_iter = iterpeek + + +class itermod(iterpeek): + """ + An iterator object that supports modifying items as they are returned. >>> a = [" A list ", ... " of strings ", @@ -181,7 +193,7 @@ class modify_iter(peek_iter): ... " extra ", ... " whitespace. "] >>> modifier = lambda s: s.strip().replace('with', 'without') - >>> for s in modify_iter(a, modifier=modifier): + >>> for s in itermod(a, modifier=modifier): ... print('"%s"' % s) "A list" "of strings" @@ -218,29 +230,30 @@ class modify_iter(peek_iter): the item. Values returned by `peek` as well as `next` are affected by - `modifier`. However, `modify_iter.sentinel` is never passed through + `modifier`. However, `itermod.sentinel` is never passed through `modifier`; it will always be returned from `peek` unmodified. """ + def __init__(self, *args, **kwargs): """__init__(o, sentinel=None, modifier=lambda x: x)""" - if 'modifier' in kwargs: - self.modifier = kwargs['modifier'] + if "modifier" in kwargs: + self.modifier = kwargs["modifier"] elif len(args) > 2: self.modifier = args[2] args = args[:2] else: self.modifier = lambda x: x if not six.callable(self.modifier): - raise TypeError('modify_iter(o, modifier): ' - 'modifier must be callable') - super(modify_iter, self).__init__(*args) + raise TypeError("itermod(o, modifier): modifier must be callable") + super(itermod, self).__init__(*args) def _fillcache(self, n): - """Cache `n` modified items. If `n` is 0 or None, 1 item is cached. + """ + Cache `n` modified items. If `n` is 0 or None, 1 item is cached. Each item returned by the iterator is passed through the - `modify_iter.modified` function before being cached. + `itermod.modified` function before being cached. """ if not n: @@ -251,3 +264,7 @@ class modify_iter(peek_iter): except StopIteration: while len(self._cache) < n: self._cache.append(self.sentinel) + + +# Backwards compatibility +modify_iter = itermod diff --git a/python/helpers/pockets/logging.py b/python/helpers/pockets/logging.py new file mode 100644 index 000000000000..076c2b1eba2f --- /dev/null +++ b/python/helpers/pockets/logging.py @@ -0,0 +1,322 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 the Pockets team, see AUTHORS. +# Licensed under the BSD License, see LICENSE for details. + +""" +A pocket full of useful logging tools! + +The `pockets.logging` module adds a `logging.TRACE` level and a +`logging.Logger.trace` method, so messages can be logged at a lower priority +level than `logging.DEBUG`. +""" + +from __future__ import absolute_import, print_function + +import inspect +import logging +import logging.config +import sys +from functools import wraps + +from six import text_type as unicode_type + + +__all__ = [ + "log_exceptions", + "AutoLogger", + "EagerFormattingAdapter", + "IndentMultilineLogFormatter", +] + + +TRACE = 5 +logging.addLevelName(TRACE, "TRACE") +logging.TRACE = TRACE + + +def trace(self, message, *args, **kwargs): + # Yes, _log() takes '*args' as 'args'. + self._log(TRACE, message, args, **kwargs) + + +logging.Logger.trace = trace + + +def log_exceptions(fn): + """ + Decorator that wraps a function and logs any raised exceptions. + + The exception will still be raised after being logged. Also logs the + arguments to every call at the trace level. + """ + from pockets.autolog import log + + @wraps(fn) + def wrapper(*args, **kwargs): + try: + a = [str(x)[:255] for x in args] + kw = dict([(k[:255], str(v)[:255]) for k, v in kwargs.items()]) + log.trace("Calling %s.%s %r %r", fn.__module__, fn.__name__, a, kw) + return fn(*args, **kwargs) + except Exception as e: + log.error("Error calling function %s: %s" % (fn.__name__, e)) + log.exception(e) + raise + + return wrapper + + +class AutoLogger(object): + """ + A logger proxy object with all of the methods and attributes of `Logger`. + + When an attribute (e.g., "debug") is requested, inspects the stack for the + calling module's name, and passes that name to `logging.getLogger`. + + What this means is that you can instantiate an `AutoLogger` anywhere, and + when you call it, the log entry shows the module where you called it, not + where it was created. + + `AutoLogger` also inspects the local variables where it is called, looking + for `self`. If `self` exists, its classname is added to the module name. + + Args: + adapter_class (LoggerAdapter): optional `LoggerAdapter` class to use. + adapter_args (list): optional args to use when instantiating an + instance of `adapter_class`. + adapter_kwargs (dict): optional kwargs to use when instantiating an + instance of `adapter_class`. + """ + + def __init__( + self, adapter_class=None, adapter_args=None, adapter_kwargs=None + ): + if adapter_args is None: + adapter_args = [] + if adapter_kwargs is None: + adapter_kwargs = {} + + self.adapter_class = adapter_class + self.adapter_args = adapter_args + self.adapter_kwargs = adapter_kwargs + + def __getattr__(self, name): + f_locals = inspect.currentframe().f_back.f_locals + if "self" in f_locals and f_locals["self"] is not None: + other = f_locals["self"] + caller_name = "%s.%s" % ( + other.__class__.__module__, + other.__class__.__name__, + ) + else: + caller_name = inspect.currentframe().f_back.f_globals["__name__"] + logger = logging.getLogger(caller_name) + + if self.adapter_class: + logger = self.adapter_class( + logger, *self.adapter_args, **self.adapter_kwargs + ) + + return getattr(logger, name) + + +class EagerFormattingAdapter(logging.LoggerAdapter): + """ + A `LoggerAdapter` that immediately interpolates message arguments if the + appropriate loglevel is set. + + This is useful because many log handlers generate log output on a separate + thread, and the value of the log arguments may have changed by the time + the handler interpolates them. This can lead to confusion when debugging + difficult bugs, as the log output will not reflect what was actually + happening when the log message was originally generated. + + For performance reasons, the interpolation ONLY happens if the appropriate + loglevel is set. This prevents unnecessary string formatting on log + messages that will just be thrown out anyway. + + Args: + logger (Logger): The underlying Logger instance to use. + extra (dict): Extra args, ignored by this implementation. + """ + + def __init__(self, logger, extra=None): + """ + Initialize the adapter with a logger and a dict-like object which + provides contextual information. This constructor signature allows + easy stacking of LoggerAdapters, if so desired. + + You can effectively pass keyword arguments as shown in the + following example:: + + adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) + + """ + self.logger = logger + self.extra = extra + + def _eagerFormat(self, msg, level, args): + """ + Eagerly apply log formatting if the appropriate level is enabled. + + Otherwise we just drop the log message (and return a string indicating + that it was suppreseed). + """ + if not hasattr(self, "isEnabledFor") or self.isEnabledFor(level): + # Do the string formatting immediately. + if args: + return self._getUnterpolatedMessage(msg, args) + else: + return msg + else: + # Otherwise, just drop the message completely to avoid anything + # going wrong in the future. This text shoudl clue one in to + # what's going on in the bizarre edge case where this ever does + # show up. + return "(log message suppressed due to insufficient log level)" + + def _getUnterpolatedMessage(self, msg, args): + """ + Returns the formatted string, will first attempt str.format and will + fallback to msg % args as it was originally. + + This is lifted almost wholesale from logging_unterpolation. + """ + original_msg = msg + + try: + msg = msg.format(*args) + except UnicodeEncodeError: + # This is most likely due to formatting a non-ascii string argument + # into a bytestring, which the %-operator automatically handles + # by casting the left side (the "msg" variable) in this context + # to unicode. So we'll do that here + # + # Handle the attempt to print utf-8 encoded data, similar to + # %-interpolation's handling of unicode formatting non-ascii + # strings + msg = unicode_type(msg).format(*args) + + except ValueError: + # From PEP-3101, value errors are of the type raised by the format + # method itself, so see if we should fall back to original + # formatting since there was an issue + if "%" in msg: + msg = msg % args + else: + # We should NOT fall back, since there's no possible string + # interpolation happening and we want a meaningful error + # message + raise + + if msg == original_msg and "%" in msg: + # There must have been no string formatting methods used, given + # the presence of args without a change in the msg + if len(args) == 1 and isinstance(args[0], dict): + # Handles cases like: + # logging.debug("a %(a)d b %(b)s", {'a':1, 'b':2}) + msg = msg % args[0] + else: + # Fall back to original formatting + msg = msg % args + + return msg + + def trace(self, msg, *args, **kwargs): + """ + Delegate a trace call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.TRACE, msg, *args, **kwargs) + + def debug(self, msg, *args, **kwargs): + """ + Delegate a debug call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.DEBUG, msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + """ + Delegate an info call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.INFO, msg, *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + """ + Delegate a warning call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.WARN, msg, *args, **kwargs) + + def warning(self, msg, *args, **kwargs): + """ + Delegate a warning call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.WARNING, msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + """ + Delegate an error call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.ERROR, msg, *args, **kwargs) + + def exception(self, msg, *args, **kwargs): + """ + Delegate an exception call to the underlying logger, after adding + contextual information from this adapter instance. + """ + kwargs["exc_info"] = 1 + self.log(logging.ERROR, msg, *args, **kwargs) + + def critical(self, msg, *args, **kwargs): + """ + Delegate a critical call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.CRITICAL, msg, *args, **kwargs) + + def fatal(self, msg, *args, **kwargs): + """ + Delegate a fatal call to the underlying logger, after adding + contextual information from this adapter instance. + """ + self.log(logging.FATAL, msg, *args, **kwargs) + + def log(self, level, msg, *args, **kwargs): + """ + Delegate a log call to the underlying logger, after adding + contextual information from this adapter instance. + """ + msg, kwargs = self.process(msg, kwargs) + # We explicitly do not pass the args into the log method here, since + # they should be "used up" by the eagerFormat method. + self.logger.log(level, self._eagerFormat(msg, level, args), **kwargs) + + +class IndentMultilineLogFormatter(logging.Formatter): + """ + Formatter which indents messages that are split across multiple lines. + + Indents all lines that start with a newline so they are easier for + external log programs to parse. + """ + + def format(self, record): + """ + Formats the given `LogRecord` by indenting all newlines. + + Args: + record (LogRecord): The `LogRecord` to format. + + Returns: + str: The formatted message with all newlines indented. + """ + if sys.version_info < (2, 7): + s = logging.Formatter.format(self, record) + else: + s = super(IndentMultilineLogFormatter, self).format(record) + return s.rstrip("\n").replace("\n", "\n ") diff --git a/python/helpers/pockets/string.py b/python/helpers/pockets/string.py index eb8873e0725d..9fce4651956f 100644 --- a/python/helpers/pockets/string.py +++ b/python/helpers/pockets/string.py @@ -1,54 +1,72 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2016 the Pockets team, see AUTHORS. +# Copyright (c) 2018 the Pockets team, see AUTHORS. # Licensed under the BSD License, see LICENSE for details. -"""A pocket full of useful string manipulation functions!""" +"""A pocket full of useful string manipulation tools!""" + +from __future__ import absolute_import, print_function -from __future__ import absolute_import import re + import six -import sys -from pockets.collections import listify +from pockets.collections import is_listy, listify + + +__all__ = [ + "camel", + "uncamel", + "fieldify", + "unfieldify", + "sluggify", + "splitcaps", + "splitify", + "UnicodeMixin", +] -__all__ = ["camel", "uncamel", "splitcaps"] # Default regular expression flags if six.PY2: - _re_flags = re.L | re.M | re.U + RE_FLAGS = re.L | re.M | re.U else: - _re_flags = re.M | re.U + RE_FLAGS = re.M | re.U -_whitespace_group_re = re.compile("(\s+)", _re_flags) +RE_NONWORD = re.compile(r"[\W_]+") -_uncamel_re = re.compile( - "(" # The whole expression is in a single group +RE_SPLITCAPS = re.compile( # Clause 1 - "(?<=[^\sA-Z])" # Preceded by neither a space nor a capital letter - "[A-Z]+[^a-z\s]*" # All non-lowercase beginning with a capital letter - "(?=[A-Z][^A-Z\s]*?[a-z]|\s|$)" # Followed by a capitalized word - "|" + r"[A-Z]+[^a-z]*" # All non-lowercase beginning with a capital letter + r"(?=[A-Z][^A-Z]*?[a-z]|$)" # Followed by a capitalized word + r"|" # Clause 2 - "(?<=[^\s])" # Preceded by a character that is not a space - "[A-Z][^A-Z\s]*?[a-z]+[^A-Z\s]*" # Capitalized word - ")", _re_flags) - -_splitcaps_re = re.compile( - # Clause 1 - "[A-Z]+[^a-z]*" # All non-lowercase beginning with a capital letter - "(?=[A-Z][^A-Z]*?[a-z]|$)" # Followed by a capitalized word - "|" - # Clause 2 - "[A-Z][^A-Z]*?[a-z]+[^A-Z]*" # Capitalized word - "|" + r"[A-Z][^A-Z]*?[a-z]+[^A-Z]*" r"|" # Capitalized word # Clause 3 - "[^A-Z]+", # All non-uppercase - _re_flags) + r"[^A-Z]+", # All non-uppercase + RE_FLAGS, +) + +RE_UNCAMEL = re.compile( + r"(" # The whole expression is in a single group + # Clause 1 + r"(?<=[^\sA-Z])" # Preceded by neither a space nor a capital letter + r"[A-Z]+[^a-z\s]*" # All non-lowercase beginning with a capital letter + r"(?=[A-Z][^A-Z\s]*?[a-z]|\s|$)" # Followed by a capitalized word + r"|" + # Clause 2 + r"(?<=[^\s])" # Preceded by a character that is not a space + r"[A-Z][^A-Z\s]*?[a-z]+[^A-Z\s]*" # Capitalized word + r")", + RE_FLAGS, +) + +RE_WHITESPACE_GROUP = re.compile(r"(\s+)", RE_FLAGS) -def camel(s, sep="_", lower_initial=False, upper_segments=None, - preserve_upper=False): - """Convert underscore_separated string (aka snake_case) to CamelCase. +def camel( + s, sep="_", lower_initial=False, upper_segments=None, preserve_upper=False +): + """ + Convert underscore_separated string (aka snake_case) to CamelCase. Works on full sentences as well as individual words: @@ -123,7 +141,7 @@ def camel(s, sep="_", lower_initial=False, upper_segments=None, lower_initial = listify(lower_initial) upper_segments = listify(upper_segments) result = [] - for word in _whitespace_group_re.split(s): + for word in RE_WHITESPACE_GROUP.split(s): segments = [segment for segment in word.split(sep) if segment] count = len(segments) for i, segment in enumerate(segments): @@ -149,7 +167,8 @@ def camel(s, sep="_", lower_initial=False, upper_segments=None, def uncamel(s, sep="_"): - """Convert CamelCase string to underscore_separated (aka snake_case). + """ + Convert CamelCase string to underscore_separated (aka snake_case). A CamelCase word is considered to be any uppercase letter followed by zero or more lowercase letters. Contiguous groups of uppercase letters – like @@ -184,11 +203,88 @@ def uncamel(s, sep="_"): str: uncamel_cased version of `s`. """ - return _uncamel_re.sub(r'{0}\1'.format(sep), s).lower() + return RE_UNCAMEL.sub(r"{0}\1".format(sep), s).lower() + + +def fieldify(s, sep="_"): + """ + Convert a string into a valid "field-like" variable name. + + Converts `s` from camel case to underscores, and replaces all spaces and + non-word characters with `sep`: + + >>> fieldify('The XmlHTTPRequest Contained, "DATA..."') + 'the_xml_http_request_contained_data' + + Args: + s (str): The string to fieldify. + + sep (str): The string to use as a word separator in the returned field. + Defaults to '_'. + + Returns: + str: The field version of `s`. + + """ + if not s: + return "" + return RE_NONWORD.sub(sep, uncamel(s)).strip(sep) + + +def unfieldify(s, sep="_"): + """ + Makes a best effort to reverse the algorithm from `fieldify`. + + Replaces instances of `sep` in `s` with a space and converts the result to + title case: + + >>> unfieldify('the_xml_http_request_contained_data') + 'The Xml Http Request Contained Data' + + Args: + s (str): The string to fieldify. + + sep (str): The string to consider a word separator in `s`. + Defaults to '_'. + + Returns: + str: The unfieldified version of `s`. + + """ + if not s: + return "" + s = s.strip(r"{0} ".format(sep)) + return (" ".join([w for w in s.split(sep) if w])).title() + + +def sluggify(s, sep="-"): + """ + Convert a string into a "slug" suitable for use in a URL. + + Converts `s` to lower case, and replaces all spaces and non-word + characters with `sep`: + + >>> sluggify('The ANGRY Wizard Shouted, "HEY..."') + 'the-angry-wizard-shouted-hey' + + Args: + s (str): The string to convert into a slug. + + sep (str): The string to use as a word separator in the slug. + Defaults to '-'. + + Returns: + str: The sluggify version of `s`. + + """ + if not s: + return "" + return RE_NONWORD.sub(sep, s).lower().strip(sep) def splitcaps(s, pattern=None, maxsplit=None, flags=0): - """Intelligently split a string on capitalized words. + """ + Intelligently split a string on capitalized words. A capitalized word is considered to be any uppercase letter followed by zero or more lowercase letters. Contiguous groups of uppercase letters – @@ -215,13 +311,13 @@ def splitcaps(s, pattern=None, maxsplit=None, flags=0): ['lower case words'] Does not split on whitespace by default. To also split - on whitespace, pass "\\\s+" for `pattern`: + on whitespace, pass "\\\\s+" for `pattern`: >>> splitcaps("Without whiteSpace pattern") ['Without white', 'Space pattern'] - >>> splitcaps("With whiteSpace pattern", pattern="\s+") + >>> splitcaps("With whiteSpace pattern", pattern=r"\\s+") ['With', 'white', 'Space', 'pattern'] - >>> splitcaps("With whiteSpace group", pattern="(\s+)") + >>> splitcaps("With whiteSpace group", pattern=r"(\\s+)") ['With', ' ', 'white', 'Space', ' ', 'group'] Args: @@ -254,13 +350,13 @@ def splitcaps(s, pattern=None, maxsplit=None, flags=0): maxsplit = -1 if pattern: - pattern_re = re.compile(pattern, flags or _re_flags) + pattern_re = re.compile(pattern, flags or RE_FLAGS) else: pattern_re = None result = [] post_maxsplit = [] - for m in _splitcaps_re.finditer(s): + for m in RE_SPLITCAPS.finditer(s): if pattern_re: for segment in pattern_re.split(m.group()): if segment: @@ -273,8 +369,8 @@ def splitcaps(s, pattern=None, maxsplit=None, flags=0): if maxsplit > 0 and len(result) >= maxsplit: if m.end() < len(s): - post_maxsplit.append(s[m.end():]) - post_maxsplit = ''.join(post_maxsplit) + post_maxsplit.append(s[m.end() :]) + post_maxsplit = "".join(post_maxsplit) if post_maxsplit: result.append(post_maxsplit) break @@ -282,9 +378,53 @@ def splitcaps(s, pattern=None, maxsplit=None, flags=0): return result if len(result) > 0 else [s] +def splitify(value, separator=",", strip=True, include_empty=False): + """ + Convert a value to a list using a supercharged `split()`. + + If `value` is a string, it is split by `separator`. If `separator` is + `None` or empty, no attempt to split is made, and `value` is returned as + the only item in a list. + + If `strip` is `True`, then the split strings will be stripped of + whitespace. If `strip` is a string, then the split strings will be + stripped of the given string. + + If `include_empty` is `False`, then empty split strings will not be + included in the returned list. + + If `value` is `None` an empty list is returned. + + If `value` is already "listy", it is returned as-is. + + If `value` is any other type, it is returned as the only item in a list. + + >>> splitify("first item, second item") + ['first item', 'second item'] + >>> splitify("first path: second path: :skipped empty path", ":") + ['first path', 'second path', 'skipped empty path'] + >>> splitify(["already", "split"]) + ['already', 'split'] + >>> splitify(None) + [] + >>> splitify(1969) + [1969] + """ + if is_listy(value): + return value + + if isinstance(value, str) and separator: + parts = value.split(separator) + if strip: + strip = None if strip is True else strip + parts = [s.strip(strip) for s in parts] + return [s for s in parts if include_empty or s] + return listify(value) + + class UnicodeMixin(object): - """Mixin class to define the proper __str__/__unicode__ methods in - Python 2 or 3. + """ + Mixin class to define proper __str__/__unicode__ methods in Python 2 or 3. Originally found on the `Porting Python 2 Code to Python 3 HOWTO`_. @@ -293,9 +433,12 @@ class UnicodeMixin(object): """ - if sys.version_info[0] >= 3: # Python 3 + if six.PY2: + + def __str__(self): + return self.__unicode__().encode("utf8") + + else: + def __str__(self): return self.__unicode__() - else: # Python 2 - def __str__(self): - return self.__unicode__().encode('utf8') diff --git a/python/helpers/sphinxcontrib/__init__.py b/python/helpers/sphinxcontrib/__init__.py index ce7beff5d83a..79bf786ebc8b 100644 --- a/python/helpers/sphinxcontrib/__init__.py +++ b/python/helpers/sphinxcontrib/__init__.py @@ -6,7 +6,7 @@ This package is a namespace package that contains all extensions distributed in the ``sphinx-contrib`` distribution. - :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. + :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/python/helpers/sphinxcontrib/napoleon/__init__.py b/python/helpers/sphinxcontrib/napoleon/__init__.py index f76efd191338..dd2a098fa919 100644 --- a/python/helpers/sphinxcontrib/napoleon/__init__.py +++ b/python/helpers/sphinxcontrib/napoleon/__init__.py @@ -1,57 +1,79 @@ # -*- coding: utf-8 -*- -# Copyright 2014 Rob Ruana -# Licensed under the BSD License, see LICENSE file for details. +""" + sphinxcontrib.napoleon + ~~~~~~~~~~~~~~~~~~~~~~ -"""Sphinx napoleon extension -- support for NumPy and Google style docstrings. + Support for NumPy and Google style docstrings. + + :copyright: Copyright 2013-2018 by Rob Ruana, see AUTHORS. + :license: BSD, see LICENSE for details. """ -import sys - -from six import iteritems -from sphinxcontrib.napoleon.docstring import GoogleDocstring, NumpyDocstring from sphinxcontrib.napoleon._version import __version__ -assert __version__ # silence pyflakes +from sphinxcontrib.napoleon.docstring import GoogleDocstring, NumpyDocstring + +if False: + # For type annotation + from typing import Any, Dict, List # NOQA -class Config(object): +class Config: """Sphinx napoleon extension settings in `conf.py`. Listed below are all the settings used by napoleon and their default values. These settings can be changed in the Sphinx `conf.py` file. Make - sure that both "sphinx.ext.autodoc" and "sphinxcontrib.napoleon" are - enabled in `conf.py`:: + sure that "sphinxcontrib.napoleon" is enabled in `conf.py`:: # conf.py # Add any Sphinx extension module names here, as strings - extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon'] + extensions = ['sphinxcontrib.napoleon'] # Napoleon settings napoleon_google_docstring = True napoleon_numpy_docstring = True + napoleon_include_init_with_doc = False napoleon_include_private_with_doc = False - napoleon_include_special_with_doc = True + napoleon_include_special_with_doc = False napoleon_use_admonition_for_examples = False napoleon_use_admonition_for_notes = False napoleon_use_admonition_for_references = False napoleon_use_ivar = False napoleon_use_param = True napoleon_use_rtype = True + napoleon_use_keyword = True + napoleon_custom_sections = None .. _Google style: - http://google.github.io/styleguide/pyguide.html + https://google.github.io/styleguide/pyguide.html .. _NumPy style: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt Attributes ---------- - napoleon_google_docstring : bool, defaults to True + napoleon_google_docstring : :obj:`bool` (Defaults to True) True to parse `Google style`_ docstrings. False to disable support for Google style docstrings. - napoleon_numpy_docstring : bool, defaults to True + napoleon_numpy_docstring : :obj:`bool` (Defaults to True) True to parse `NumPy style`_ docstrings. False to disable support for NumPy style docstrings. - napoleon_include_private_with_doc : bool, defaults to False + napoleon_include_init_with_doc : :obj:`bool` (Defaults to False) + True to list ``__init___`` docstrings separately from the class + docstring. False to fall back to Sphinx's default behavior, which + considers the ``__init___`` docstring as part of the class + documentation. + + **If True**:: + + def __init__(self): + \"\"\" + This will be included in the docs because it has a docstring + \"\"\" + + def __init__(self): + # This will NOT be included in the docs + + napoleon_include_private_with_doc : :obj:`bool` (Defaults to False) True to include private members (like ``_membername``) with docstrings in the documentation. False to fall back to Sphinx's default behavior. @@ -67,7 +89,7 @@ class Config(object): # This will NOT be included in the docs pass - napoleon_include_special_with_doc : bool, defaults to True + napoleon_include_special_with_doc : :obj:`bool` (Defaults to False) True to include special members (like ``__membername__``) with docstrings in the documentation. False to fall back to Sphinx's default behavior. @@ -84,7 +106,7 @@ class Config(object): # This will NOT be included in the docs return unicode(self.__class__.__name__) - napoleon_use_admonition_for_examples : bool, defaults to False + napoleon_use_admonition_for_examples : :obj:`bool` (Defaults to False) True to use the ``.. admonition::`` directive for the **Example** and **Examples** sections. False to use the ``.. rubric::`` directive instead. One may look better than the other depending on what HTML @@ -108,7 +130,7 @@ class Config(object): This is just a quick example - napoleon_use_admonition_for_notes : bool, defaults to False + napoleon_use_admonition_for_notes : :obj:`bool` (Defaults to False) True to use the ``.. admonition::`` directive for **Notes** sections. False to use the ``.. rubric::`` directive instead. @@ -121,7 +143,7 @@ class Config(object): -------- :attr:`napoleon_use_admonition_for_examples` - napoleon_use_admonition_for_references : bool, defaults to False + napoleon_use_admonition_for_references : :obj:`bool` (Defaults to False) True to use the ``.. admonition::`` directive for **References** sections. False to use the ``.. rubric::`` directive instead. @@ -129,7 +151,7 @@ class Config(object): -------- :attr:`napoleon_use_admonition_for_examples` - napoleon_use_ivar : bool, defaults to False + napoleon_use_ivar : :obj:`bool` (Defaults to False) True to use the ``:ivar:`` role for instance variables. False to use the ``.. attribute::`` directive instead. @@ -149,11 +171,11 @@ class Config(object): .. attribute:: attr1 - *int* - Description of `attr1` - napoleon_use_param : bool, defaults to True + :type: int + + napoleon_use_param : :obj:`bool` (Defaults to True) True to use a ``:param:`` role for each function parameter. False to use a single ``:parameters:`` role for all the parameters. @@ -180,7 +202,22 @@ class Config(object): * **arg2** (*int, optional*) -- Description of `arg2`, defaults to 0 - napoleon_use_rtype : bool, defaults to True + napoleon_use_keyword : :obj:`bool` (Defaults to True) + True to use a ``:keyword:`` role for each function keyword argument. + False to use a single ``:keyword arguments:`` role for all the + keywords. + + This behaves similarly to :attr:`napoleon_use_param`. Note unlike + docutils, ``:keyword:`` and ``:param:`` will not be treated the same + way - there will be a separate "Keyword Arguments" section, rendered + in the same fashion as "Parameters" section (type links created if + possible) + + See Also + -------- + :attr:`napoleon_use_param` + + napoleon_use_rtype : :obj:`bool` (Defaults to True) True to use the ``:rtype:`` role for the return type. False to output the return type inline with the description. @@ -200,28 +237,46 @@ class Config(object): :returns: *bool* -- True if successful, False otherwise + napoleon_custom_sections : :obj:`list` (Defaults to None) + Add a list of custom sections to include, expanding the list of parsed sections. + + The entries can either be strings or tuples, depending on the intention: + * To create a custom "generic" section, just pass a string. + * To create an alias for an existing section, pass a tuple containing the + alias name and the original, in that order. + + If an entry is just a string, it is interpreted as a header for a generic + section. If the entry is a tuple/list/indexed container, the first entry + is the name of the section, the second is the section key to emulate. + + """ _config_values = { 'napoleon_google_docstring': (True, 'env'), 'napoleon_numpy_docstring': (True, 'env'), + 'napoleon_include_init_with_doc': (False, 'env'), 'napoleon_include_private_with_doc': (False, 'env'), - 'napoleon_include_special_with_doc': (True, 'env'), + 'napoleon_include_special_with_doc': (False, 'env'), 'napoleon_use_admonition_for_examples': (False, 'env'), 'napoleon_use_admonition_for_notes': (False, 'env'), 'napoleon_use_admonition_for_references': (False, 'env'), 'napoleon_use_ivar': (False, 'env'), 'napoleon_use_param': (True, 'env'), 'napoleon_use_rtype': (True, 'env'), + 'napoleon_use_keyword': (True, 'env'), + 'napoleon_custom_sections': (None, 'env') } def __init__(self, **settings): - for name, (default, rebuild) in iteritems(self._config_values): + # type: (Any) -> None + for name, (default, rebuild) in self._config_values.items(): setattr(self, name, default) - for name, value in iteritems(settings): + for name, value in settings.items(): setattr(self, name, value) def setup(app): + # type: (Sphinx) -> Dict[unicode, Any] """Sphinx extension setup function. When the extension is loaded, Sphinx imports this module and executes @@ -242,20 +297,45 @@ def setup(app): `The Extension API `_ - """ from sphinx.application import Sphinx if not isinstance(app, Sphinx): - return # probably called by tests + # probably called by tests + return {'version': __version__, 'parallel_read_safe': True} + _patch_python_domain() + + app.setup_extension('sphinx.ext.autodoc') app.connect('autodoc-process-docstring', _process_docstring) app.connect('autodoc-skip-member', _skip_member) - for name, (default, rebuild) in iteritems(Config._config_values): + for name, (default, rebuild) in Config._config_values.items(): app.add_config_value(name, default, rebuild) + return {'version': __version__, 'parallel_read_safe': True} + + +def _patch_python_domain(): + # type: () -> None + try: + from sphinx.domains.python import PyTypedField + except ImportError: + pass + else: + import sphinx.domains.python + from sphinx.locale import _ + for doc_field in sphinx.domains.python.PyObject.doc_field_types: + if doc_field.name == 'parameter': + doc_field.names = ('param', 'parameter', 'arg', 'argument') + break + sphinx.domains.python.PyObject.doc_field_types.append( + PyTypedField('keyword', label=_('Keyword Arguments'), + names=('keyword', 'kwarg', 'kwparam'), + typerolename='obj', typenames=('paramtype', 'kwtype'), + can_collapse=True)) def _process_docstring(app, what, name, obj, options, lines): + # type: (Sphinx, unicode, unicode, Any, Any, List[unicode]) -> None """Process the docstring for a given python object. Called when autodoc has read and processed a docstring. `lines` is a list @@ -292,6 +372,7 @@ def _process_docstring(app, what, name, obj, options, lines): """ result_lines = lines + docstring = None # type: GoogleDocstring if app.config.napoleon_numpy_docstring: docstring = NumpyDocstring(result_lines, app.config, app, what, name, obj, options) @@ -304,11 +385,14 @@ def _process_docstring(app, what, name, obj, options, lines): def _skip_member(app, what, name, obj, skip, options): + # type: (Sphinx, unicode, unicode, Any, bool, Any) -> bool """Determine if private and special class members are included in docs. The following settings in conf.py determine if private and special class - members are included in the generated documentation: + members or init methods are included in the generated documentation: + * ``napoleon_include_init_with_doc`` -- + include init methods if they have docstrings * ``napoleon_include_private_with_doc`` -- include private members if they have docstrings * ``napoleon_include_special_with_doc`` -- @@ -345,43 +429,47 @@ def _skip_member(app, what, name, obj, skip, options): """ has_doc = getattr(obj, '__doc__', False) is_member = (what == 'class' or what == 'exception' or what == 'module') - if name != '__weakref__' and name != '__init__' and has_doc and is_member: + if name != '__weakref__' and has_doc and is_member: cls_is_owner = False - if what == 'class' or what == 'exception': - if sys.version_info[0] < 3: - cls = getattr(obj, 'im_class', getattr(obj, '__objclass__', - None)) - cls_is_owner = (cls and hasattr(cls, name) and - name in cls.__dict__) - elif sys.version_info[1] >= 3: - qualname = getattr(obj, '__qualname__', '') - cls_path, _, _ = qualname.rpartition('.') - if cls_path: - try: - if '.' in cls_path: - import importlib - import functools + import six + if six.PY2 and (what == 'class' or what == 'exception'): + cls = getattr(obj, 'im_class', getattr(obj, '__objclass__', + None)) + cls_is_owner = (cls and hasattr(cls, name) and + name in cls.__dict__) + elif what == 'class' or what == 'exception': + qualname = getattr(obj, '__qualname__', '') + cls_path, _, _ = qualname.rpartition('.') + if cls_path: + try: + if '.' in cls_path: + import importlib + import functools - mod = importlib.import_module(obj.__module__) - mod_path = cls_path.split('.') - cls = functools.reduce(getattr, mod_path, mod) - else: - cls = obj.__globals__[cls_path] - except: - cls_is_owner = False + mod = importlib.import_module(obj.__module__) + mod_path = cls_path.split('.') + cls = functools.reduce(getattr, mod_path, mod) else: - cls_is_owner = (cls and hasattr(cls, name) and - name in cls.__dict__) - else: + cls = obj.__globals__[cls_path] + except Exception: cls_is_owner = False + else: + cls_is_owner = (cls and hasattr(cls, name) and # type: ignore + name in cls.__dict__) else: - cls_is_owner = True + cls_is_owner = False if what == 'module' or cls_is_owner: - is_special = name.startswith('__') and name.endswith('__') - is_private = not is_special and name.startswith('_') + is_init = (name == '__init__') + is_special = (not is_init and name.startswith('__') and + name.endswith('__')) + is_private = (not is_init and not is_special and + name.startswith('_')) + inc_init = app.config.napoleon_include_init_with_doc inc_special = app.config.napoleon_include_special_with_doc inc_private = app.config.napoleon_include_private_with_doc - if (is_special and inc_special) or (is_private and inc_private): + if ((is_special and inc_special) or + (is_private and inc_private) or + (is_init and inc_init)): return False - return skip + return None diff --git a/python/helpers/sphinxcontrib/napoleon/_upstream.py b/python/helpers/sphinxcontrib/napoleon/_upstream.py new file mode 100644 index 000000000000..7fd7c2446956 --- /dev/null +++ b/python/helpers/sphinxcontrib/napoleon/_upstream.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" + sphinxcontrib.napoleon._upstream + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Functions to help compatibility with upstream sphinx.ext.napoleon. + + :copyright: Copyright 2013-2018 by Rob Ruana, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +# Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. + +def _(message, *args): + """ + NOOP implementation of sphinx.locale.get_translation shortcut. + """ + return message diff --git a/python/helpers/sphinxcontrib/napoleon/_version.py b/python/helpers/sphinxcontrib/napoleon/_version.py index d305fdb86e77..c8d2b48ad929 100644 --- a/python/helpers/sphinxcontrib/napoleon/_version.py +++ b/python/helpers/sphinxcontrib/napoleon/_version.py @@ -5,4 +5,4 @@ # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module -__version__ = '0.3.11' +__version__ = '0.7' diff --git a/python/helpers/sphinxcontrib/napoleon/docstring.py b/python/helpers/sphinxcontrib/napoleon/docstring.py index e4694ac8419b..28824ae8432d 100644 --- a/python/helpers/sphinxcontrib/napoleon/docstring.py +++ b/python/helpers/sphinxcontrib/napoleon/docstring.py @@ -1,60 +1,83 @@ # -*- coding: utf-8 -*- -# Copyright 2014 Rob Ruana -# Licensed under the BSD License, see LICENSE file for details. +""" + sphinxcontrib.napoleon.docstring + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -"""Classes for docstring parsing and formatting.""" -import collections + Classes for docstring parsing and formatting. + + + :copyright: Copyright 2013-2018 by Rob Ruana, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + import inspect import re -import sys +try: + from collections.abc import Callable +except ImportError: + from collections import Callable +from functools import partial + +from six import string_types, u from six.moves import range -from pockets import modify_iter -from six import string_types +from pockets import modify_iter, UnicodeMixin +from sphinxcontrib.napoleon._upstream import _ + +if False: + # For type annotation + from typing import Any, Dict, List, Tuple, Type, Union # NOQA + from sphinx.application import Sphinx # NOQA + from sphinx.config import Config as SphinxConfig # NOQA + _directive_regex = re.compile(r'\.\. \S+::') _google_section_regex = re.compile(r'^(\s|\w)+:\s*$') -_google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.+?)\s*\)') +_google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)') _numpy_section_regex = re.compile(r'^[=\-`:\'"~^_*+#<>]{2,}\s*$') -_xref_regex = re.compile(r'(:\w+:\S+:`.+?`|:\S+:`.+?`|`.+?`)') +_single_colon_regex = re.compile(r'(?\()?' + r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])' + r'(?(paren)\)|\.)(\s+\S|\s*$)') -class GoogleDocstring(object): +class GoogleDocstring(UnicodeMixin): """Convert Google style docstrings to reStructuredText. Parameters ---------- - docstring : str or List[str] + docstring : :obj:`str` or :obj:`list` of :obj:`str` The docstring to parse, given either as a string or split into individual lines. - config : Optional[sphinxcontrib.napoleon.Config or sphinx.config.Config] + config: :obj:`sphinxcontrib.napoleon.Config` or :obj:`sphinx.config.Config` The configuration settings to use. If not given, defaults to the config object on `app`; or if `app` is not given defaults to the - a new `sphinxcontrib.napoleon.Config` object. + a new :class:`sphinxcontrib.napoleon.Config` object. - See Also - -------- - :class:`sphinxcontrib.napoleon.Config` Other Parameters ---------------- - app : Optional[sphinx.application.Sphinx] + app : :class:`sphinx.application.Sphinx`, optional Application object representing the Sphinx process. - what : Optional[str] + what : :obj:`str`, optional A string specifying the type of the object to which the docstring belongs. Valid values: "module", "class", "exception", "function", "method", "attribute". - name : Optional[str] + name : :obj:`str`, optional The fully qualified name of the object. obj : module, class, exception, function, method, or attribute The object to which the docstring belongs. - options : Optional[sphinx.ext.autodoc.Options] + options : :class:`sphinx.ext.autodoc.Options`, optional The options given to the directive: an object with attributes inherited_members, undoc_members, show_inheritance and noindex that are True if the flag option of same name was given to the auto directive. + Example ------- >>> from sphinxcontrib.napoleon import Config @@ -84,21 +107,26 @@ class GoogleDocstring(object): """ + + _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" + r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) + def __init__(self, docstring, config=None, app=None, what='', name='', obj=None, options=None): + # type: (Union[unicode, List[unicode]], SphinxConfig, Sphinx, unicode, unicode, Any, Any) -> None # NOQA self._config = config self._app = app if not self._config: from sphinxcontrib.napoleon import Config - self._config = self._app and self._app.config or Config() + self._config = self._app and self._app.config or Config() # type: ignore if not what: if inspect.isclass(obj): what = 'class' elif inspect.ismodule(obj): what = 'module' - elif isinstance(obj, collections.Callable): + elif isinstance(obj, Callable): what = 'function' else: what = 'object' @@ -111,22 +139,28 @@ class GoogleDocstring(object): docstring = docstring.splitlines() self._lines = docstring self._line_iter = modify_iter(docstring, modifier=lambda s: s.rstrip()) - self._parsed_lines = [] + self._parsed_lines = [] # type: List[unicode] self._is_in_section = False self._section_indent = 0 if not hasattr(self, '_directive_sections'): - self._directive_sections = [] + self._directive_sections = [] # type: List[unicode] if not hasattr(self, '_sections'): self._sections = { 'args': self._parse_parameters_section, 'arguments': self._parse_parameters_section, + 'attention': partial(self._parse_admonition, 'attention'), 'attributes': self._parse_attributes_section, + 'caution': partial(self._parse_admonition, 'caution'), + 'danger': partial(self._parse_admonition, 'danger'), + 'error': partial(self._parse_admonition, 'error'), 'example': self._parse_examples_section, 'examples': self._parse_examples_section, + 'hint': partial(self._parse_admonition, 'hint'), + 'important': partial(self._parse_admonition, 'important'), 'keyword args': self._parse_keyword_arguments_section, 'keyword arguments': self._parse_keyword_arguments_section, 'methods': self._parse_methods_section, - 'note': self._parse_note_section, + 'note': partial(self._parse_admonition, 'note'), 'notes': self._parse_notes_section, 'other parameters': self._parse_other_parameters_section, 'parameters': self._parse_parameters_section, @@ -135,29 +169,21 @@ class GoogleDocstring(object): 'raises': self._parse_raises_section, 'references': self._parse_references_section, 'see also': self._parse_see_also_section, - 'warning': self._parse_warning_section, - 'warnings': self._parse_warning_section, + 'tip': partial(self._parse_admonition, 'tip'), + 'todo': partial(self._parse_admonition, 'todo'), + 'warning': partial(self._parse_admonition, 'warning'), + 'warnings': partial(self._parse_admonition, 'warning'), 'warns': self._parse_warns_section, 'yield': self._parse_yields_section, 'yields': self._parse_yields_section, - } + } # type: Dict[unicode, Callable] + + self._load_custom_sections() + self._parse() - def __str__(self): - """Return the parsed docstring in reStructuredText format. - - Returns - ------- - str - UTF-8 encoded version of the docstring. - - """ - if sys.version_info[0] >= 3: - return self.__unicode__() - else: - return self.__unicode__().encode('utf8') - def __unicode__(self): + # type: () -> unicode """Return the parsed docstring in reStructuredText format. Returns @@ -166,20 +192,22 @@ class GoogleDocstring(object): Unicode version of the docstring. """ - return u'\n'.join(self.lines()) + return u('\n').join(self.lines()) def lines(self): + # type: () -> List[unicode] """Return the parsed lines of the docstring in reStructuredText format. Returns ------- - List[str] + list(str) The lines of the docstring in a list. """ return self._parsed_lines def _consume_indented_block(self, indent=1): + # type: (int) -> List[unicode] lines = [] line = self._line_iter.peek() while(not self._is_section_break() and @@ -189,6 +217,7 @@ class GoogleDocstring(object): return lines def _consume_contiguous(self): + # type: () -> List[unicode] lines = [] while (self._line_iter.has_next() and self._line_iter.peek() and @@ -197,6 +226,7 @@ class GoogleDocstring(object): return lines def _consume_empty(self): + # type: () -> List[unicode] lines = [] line = self._line_iter.peek() while self._line_iter.has_next() and not line: @@ -205,30 +235,29 @@ class GoogleDocstring(object): return lines def _consume_field(self, parse_type=True, prefer_type=False): + # type: (bool, bool) -> Tuple[unicode, unicode, List[unicode]] line = next(self._line_iter) before, colon, after = self._partition_field_on_colon(line) - _name, _type, _desc = before, '', after + _name, _type, _desc = before, '', after # type: unicode, unicode, unicode if parse_type: - match = _google_typed_arg_regex.match(before) + match = _google_typed_arg_regex.match(before) # type: ignore if match: _name = match.group(1) _type = match.group(2) - if _name[:2] == '**': - _name = r'\*\*'+_name[2:] - elif _name[:1] == '*': - _name = r'\*'+_name[1:] + _name = self._escape_args_and_kwargs(_name) if prefer_type and not _type: _type, _name = _name, _type indent = self._get_indent(line) + 1 - _desc = [_desc] + self._dedent(self._consume_indented_block(indent)) - _desc = self.__class__(_desc, self._config).lines() - return _name, _type, _desc + _descs = [_desc] + self._dedent(self._consume_indented_block(indent)) + _descs = self.__class__(_descs, self._config).lines() + return _name, _type, _descs def _consume_fields(self, parse_type=True, prefer_type=False): + # type: (bool, bool) -> List[Tuple[unicode, unicode, List[unicode]]] self._consume_empty() fields = [] while not self._is_section_break(): @@ -238,19 +267,22 @@ class GoogleDocstring(object): return fields def _consume_inline_attribute(self): + # type: () -> Tuple[unicode, List[unicode]] line = next(self._line_iter) _type, colon, _desc = self._partition_field_on_colon(line) - if not colon: + if not colon or not _desc: _type, _desc = _desc, _type - _desc = [_desc] + self._dedent(self._consume_to_end()) - _desc = self.__class__(_desc, self._config).lines() - return _type, _desc + _desc += colon + _descs = [_desc] + self._dedent(self._consume_to_end()) + _descs = self.__class__(_descs, self._config).lines() + return _type, _descs def _consume_returns_section(self): + # type: () -> List[Tuple[unicode, unicode, List[unicode]]] lines = self._dedent(self._consume_to_next_section()) if lines: before, colon, after = self._partition_field_on_colon(lines[0]) - _name, _type, _desc = '', '', lines + _name, _type, _desc = '', '', lines # type: unicode, unicode, List[unicode] if colon: if after: @@ -258,12 +290,7 @@ class GoogleDocstring(object): else: _desc = lines[1:] - match = _google_typed_arg_regex.match(before) - if match: - _name = match.group(1) - _type = match.group(2) - else: - _type = before + _type = before _desc = self.__class__(_desc, self._config).lines() return [(_name, _type, _desc,)] @@ -271,10 +298,12 @@ class GoogleDocstring(object): return [] def _consume_usage_section(self): + # type: () -> List[unicode] lines = self._dedent(self._consume_to_next_section()) return lines def _consume_section_header(self): + # type: () -> unicode section = next(self._line_iter) stripped_section = section.strip(':') if stripped_section.lower() in self._sections: @@ -282,12 +311,14 @@ class GoogleDocstring(object): return section def _consume_to_end(self): + # type: () -> List[unicode] lines = [] while self._line_iter.has_next(): lines.append(next(self._line_iter)) return lines def _consume_to_next_section(self): + # type: () -> List[unicode] self._consume_empty() lines = [] while not self._is_section_break(): @@ -295,23 +326,49 @@ class GoogleDocstring(object): return lines + self._consume_empty() def _dedent(self, lines, full=False): + # type: (List[unicode], bool) -> List[unicode] if full: return [line.lstrip() for line in lines] else: min_indent = self._get_min_indent(lines) return [line[min_indent:] for line in lines] + def _escape_args_and_kwargs(self, name): + # type: (unicode) -> unicode + if name[:2] == '**': + return r'\*\*' + name[2:] + elif name[:1] == '*': + return r'\*' + name[1:] + else: + return name + + def _fix_field_desc(self, desc): + # type: (List[unicode]) -> List[unicode] + if self._is_list(desc): + desc = [u''] + desc + elif desc[0].endswith('::'): + desc_block = desc[1:] + indent = self._get_indent(desc[0]) + block_indent = self._get_initial_indent(desc_block) + if block_indent > indent: + desc = [u''] + desc + else: + desc = ['', desc[0]] + self._indent(desc_block, 4) + return desc + def _format_admonition(self, admonition, lines): + # type: (unicode, List[unicode]) -> List[unicode] lines = self._strip_empty(lines) if len(lines) == 1: return ['.. %s:: %s' % (admonition, lines[0].strip()), ''] elif lines: lines = self._indent(self._dedent(lines), 3) - return ['.. %s::' % admonition, ''] + lines + [''] + return [u'.. %s::' % admonition, u''] + lines + [u''] else: - return ['.. %s::' % admonition, ''] + return [u'.. %s::' % admonition, u''] def _format_block(self, prefix, lines, padding=None): + # type: (unicode, List[unicode], unicode) -> List[unicode] if lines: if padding is None: padding = ' ' * len(prefix) @@ -327,14 +384,32 @@ class GoogleDocstring(object): else: return [prefix] + def _format_docutils_params(self, fields, field_role='param', + type_role='type'): + # type: (List[Tuple[unicode, unicode, List[unicode]]], unicode, unicode) -> List[unicode] # NOQA + lines = [] + for _name, _type, _desc in fields: + _desc = self._strip_empty(_desc) + if any(_desc): + _desc = self._fix_field_desc(_desc) + field = ':%s %s: ' % (field_role, _name) + lines.extend(self._format_block(field, _desc)) + else: + lines.append(':%s %s:' % (field_role, _name)) + + if _type: + lines.append(':%s %s: %s' % (type_role, _name, _type)) + return lines + [''] + def _format_field(self, _name, _type, _desc): + # type: (unicode, unicode, List[unicode]) -> List[unicode] _desc = self._strip_empty(_desc) has_desc = any(_desc) separator = has_desc and ' -- ' or '' if _name: if _type: if '`' in _type: - field = '**%s** (%s)%s' % (_name, _type, separator) + field = '**%s** (%s)%s' % (_name, _type, separator) # type: unicode else: field = '**%s** (*%s*)%s' % (_name, _type, separator) else: @@ -348,15 +423,20 @@ class GoogleDocstring(object): field = '' if has_desc: - return [field + _desc[0]] + _desc[1:] + _desc = self._fix_field_desc(_desc) + if _desc[0]: + return [field + _desc[0]] + _desc[1:] + else: + return [field] + _desc else: return [field] def _format_fields(self, field_type, fields): + # type: (unicode, List[Tuple[unicode, unicode, List[unicode]]]) -> List[unicode] field_type = ':%s:' % field_type.strip() padding = ' ' * len(field_type) multi = len(fields) > 1 - lines = [] + lines = [] # type: List[unicode] for _name, _type, _desc in fields: field = self._format_field(_name, _type, _desc) if multi: @@ -371,6 +451,7 @@ class GoogleDocstring(object): return lines def _get_current_indent(self, peek_ahead=0): + # type: (int) -> int line = self._line_iter.peek(peek_ahead + 1)[peek_ahead] while line != self._line_iter.sentinel: if line: @@ -380,12 +461,21 @@ class GoogleDocstring(object): return 0 def _get_indent(self, line): + # type: (unicode) -> int for i, s in enumerate(line): if not s.isspace(): return i return len(line) + def _get_initial_indent(self, lines): + # type: (List[unicode]) -> int + for line in lines: + if line: + return self._get_indent(line) + return 0 + def _get_min_indent(self, lines): + # type: (List[unicode]) -> int min_indent = None for line in lines: if line: @@ -397,9 +487,11 @@ class GoogleDocstring(object): return min_indent or 0 def _indent(self, lines, n=4): + # type: (List[unicode], int) -> List[unicode] return [(' ' * n) + line for line in lines] def _is_indented(self, line, indent=1): + # type: (unicode, int) -> bool for i, s in enumerate(line): if i >= indent: return True @@ -407,7 +499,26 @@ class GoogleDocstring(object): return False return False + def _is_list(self, lines): + # type: (List[unicode]) -> bool + if not lines: + return False + if _bullet_list_regex.match(lines[0]): # type: ignore + return True + if _enumerated_list_regex.match(lines[0]): # type: ignore + return True + if len(lines) < 2 or lines[0].endswith('::'): + return False + indent = self._get_indent(lines[0]) + next_indent = indent + for line in lines[1:]: + if line: + next_indent = self._get_indent(line) + break + return next_indent > indent + def _is_section_header(self): + # type: () -> bool section = self._line_iter.peek().lower() match = _google_section_regex.match(section) if match and section.strip(':') in self._sections: @@ -422,6 +533,7 @@ class GoogleDocstring(object): return False def _is_section_break(self): + # type: () -> bool line = self._line_iter.peek() return (not self._line_iter.has_next() or self._is_section_header() or @@ -429,11 +541,36 @@ class GoogleDocstring(object): line and not self._is_indented(line, self._section_indent))) + def _load_custom_sections(self): + # type: () -> None + + if self._config.napoleon_custom_sections is not None: + for entry in self._config.napoleon_custom_sections: + if isinstance(entry, string_types): + # if entry is just a label, add to sections list, + # using generic section logic. + self._sections[entry.lower()] = self._parse_custom_generic_section + else: + # otherwise, assume entry is container; + # [0] is new section, [1] is the section to alias. + # in the case of key mismatch, just handle as generic section. + self._sections[entry[0].lower()] = \ + self._sections.get(entry[1].lower(), + self._parse_custom_generic_section) + def _parse(self): + # type: () -> None self._parsed_lines = self._consume_empty() if self._name and (self._what == 'attribute' or self._what == 'data'): - self._parsed_lines.extend(self._parse_attribute_docstring()) + # Implicit stop using StopIteration no longer allowed in + # Python 3.7; see PEP 479 + res = [] # type: List[unicode] + try: + res = self._parse_attribute_docstring() + except StopIteration: + pass + self._parsed_lines.extend(res) return while self._line_iter.has_next(): @@ -442,7 +579,7 @@ class GoogleDocstring(object): section = self._consume_section_header() self._is_in_section = True self._section_indent = self._get_current_indent() - if _directive_regex.match(section): + if _directive_regex.match(section): # type: ignore lines = [section] + self._consume_to_next_section() else: lines = self._sections[section.lower()](section) @@ -456,43 +593,69 @@ class GoogleDocstring(object): lines = self._consume_to_next_section() self._parsed_lines.extend(lines) + def _parse_admonition(self, admonition, section): + # type (unicode, unicode) -> List[unicode] + lines = self._consume_to_next_section() + return self._format_admonition(admonition, lines) + def _parse_attribute_docstring(self): + # type: () -> List[unicode] _type, _desc = self._consume_inline_attribute() - return self._format_field('', _type, _desc) + lines = self._format_field('', '', _desc) + if _type: + lines.extend(['', ':type: %s' % _type]) + return lines def _parse_attributes_section(self, section): + # type: (unicode) -> List[unicode] lines = [] for _name, _type, _desc in self._consume_fields(): if self._config.napoleon_use_ivar: - field = ':ivar %s: ' % _name + _name = self._qualify_name(_name, self._obj) + field = ':ivar %s: ' % _name # type: unicode lines.extend(self._format_block(field, _desc)) if _type: lines.append(':vartype %s: %s' % (_name, _type)) else: lines.extend(['.. attribute:: ' + _name, '']) - field = self._format_field('', _type, _desc) - lines.extend(self._indent(field, 3)) + fields = self._format_field('', '', _desc) + lines.extend(self._indent(fields, 3)) + if _type: + lines.append('') + lines.extend(self._indent([':type: %s' % _type], 3)) lines.append('') if self._config.napoleon_use_ivar: lines.append('') return lines def _parse_examples_section(self, section): + # type: (unicode) -> List[unicode] + labels = { + 'example': _('Example'), + 'examples': _('Examples'), + } # type: Dict[unicode, unicode] use_admonition = self._config.napoleon_use_admonition_for_examples - return self._parse_generic_section(section, use_admonition) + label = labels.get(section.lower(), section) + return self._parse_generic_section(label, use_admonition) + + def _parse_custom_generic_section(self, section): + # for now, no admonition for simple custom sections + return self._parse_generic_section(section, False) def _parse_usage_section(self, section): - header = ['.. rubric:: Usage:', ''] - block = ['.. code-block:: python', ''] + # type: (unicode) -> List[unicode] + header = ['.. rubric:: Usage:', ''] # type: List[unicode] + block = ['.. code-block:: python', ''] # type: List[unicode] lines = self._consume_usage_section() lines = self._indent(lines, 3) return header + block + lines + [''] def _parse_generic_section(self, section, use_admonition): + # type: (unicode, bool) -> List[unicode] lines = self._strip_empty(self._consume_to_next_section()) lines = self._dedent(lines) if use_admonition: - header = '.. admonition:: %s' % section + header = '.. admonition:: %s' % section # type: unicode lines = self._indent(lines, 3) else: header = '.. rubric:: %s' % section @@ -502,84 +665,66 @@ class GoogleDocstring(object): return [header, ''] def _parse_keyword_arguments_section(self, section): - return self._format_fields('Keyword Arguments', self._consume_fields()) + # type: (unicode) -> List[unicode] + fields = self._consume_fields() + if self._config.napoleon_use_keyword: + return self._format_docutils_params( + fields, + field_role="keyword", + type_role="kwtype") + else: + return self._format_fields(_('Keyword Arguments'), fields) def _parse_methods_section(self, section): - lines = [] - for _name, _, _desc in self._consume_fields(parse_type=False): + # type: (unicode) -> List[unicode] + lines = [] # type: List[unicode] + for _name, _type, _desc in self._consume_fields(parse_type=False): lines.append('.. method:: %s' % _name) if _desc: - lines.extend([''] + self._indent(_desc, 3)) + lines.extend([u''] + self._indent(_desc, 3)) lines.append('') return lines - def _parse_note_section(self, section): - lines = self._consume_to_next_section() - return self._format_admonition('note', lines) - def _parse_notes_section(self, section): + # type: (unicode) -> List[unicode] use_admonition = self._config.napoleon_use_admonition_for_notes - return self._parse_generic_section('Notes', use_admonition) + return self._parse_generic_section(_('Notes'), use_admonition) def _parse_other_parameters_section(self, section): - return self._format_fields('Other Parameters', self._consume_fields()) + # type: (unicode) -> List[unicode] + return self._format_fields(_('Other Parameters'), self._consume_fields()) def _parse_parameters_section(self, section): + # type: (unicode) -> List[unicode] fields = self._consume_fields() if self._config.napoleon_use_param: - lines = [] - for _name, _type, _desc in fields: - field = ':param %s: ' % _name - lines.extend(self._format_block(field, _desc)) - if _type: - lines.append(':type %s: %s' % (_name, _type)) - return lines + [''] + return self._format_docutils_params(fields) else: - return self._format_fields('Parameters', fields) + return self._format_fields(_('Parameters'), fields) def _parse_raises_section(self, section): + # type: (unicode) -> List[unicode] fields = self._consume_fields(parse_type=False, prefer_type=True) - field_type = ':raises:' - padding = ' ' * len(field_type) - multi = len(fields) > 1 - lines = [] - for _, _type, _desc in fields: + lines = [] # type: List[unicode] + for _name, _type, _desc in fields: + m = self._name_rgx.match(_type).groupdict() # type: ignore + if m['role']: + _type = m['name'] + _type = ' ' + _type if _type else '' _desc = self._strip_empty(_desc) - has_desc = any(_desc) - separator = has_desc and ' -- ' or '' - if _type: - has_refs = '`' in _type or ':' in _type - has_space = any(c in ' \t\n\v\f ' for c in _type) - - if not has_refs and not has_space: - _type = ':exc:`%s`%s' % (_type, separator) - elif has_desc and has_space: - _type = '*%s*%s' % (_type, separator) - else: - _type = '%s%s' % (_type, separator) - - if has_desc: - field = [_type + _desc[0]] + _desc[1:] - else: - field = [_type] - else: - field = _desc - if multi: - if lines: - lines.extend(self._format_block(padding + ' * ', field)) - else: - lines.extend(self._format_block(field_type + ' * ', field)) - else: - lines.extend(self._format_block(field_type + ' ', field)) - if lines and lines[-1]: + _descs = ' ' + '\n '.join(_desc) if any(_desc) else '' + lines.append(':raises%s:%s' % (_type, _descs)) + if lines: lines.append('') return lines def _parse_references_section(self, section): + # type: (unicode) -> List[unicode] use_admonition = self._config.napoleon_use_admonition_for_references - return self._parse_generic_section('References', use_admonition) + return self._parse_generic_section(_('References'), use_admonition) def _parse_returns_section(self, section): + # type: (unicode) -> List[unicode] fields = self._consume_returns_section() multi = len(fields) > 1 if multi: @@ -587,7 +732,7 @@ class GoogleDocstring(object): else: use_rtype = self._config.napoleon_use_rtype - lines = [] + lines = [] # type: List[unicode] for _name, _type, _desc in fields: if use_rtype: field = self._format_field(_name, '', _desc) @@ -608,34 +753,34 @@ class GoogleDocstring(object): return lines def _parse_see_also_section(self, section): - lines = self._consume_to_next_section() - return self._format_admonition('seealso', lines) - - def _parse_warning_section(self, section): - lines = self._consume_to_next_section() - return self._format_admonition('warning', lines) + # type (unicode) -> List[unicode] + return self._parse_admonition('seealso', section) def _parse_warns_section(self, section): - return self._format_fields('Warns', self._consume_fields()) + # type: (unicode) -> List[unicode] + return self._format_fields(_('Warns'), self._consume_fields()) def _parse_yields_section(self, section): + # type: (unicode) -> List[unicode] fields = self._consume_returns_section() - return self._format_fields('Yields', fields) + return self._format_fields(_('Yields'), fields) def _partition_field_on_colon(self, line): + # type: (unicode) -> Tuple[unicode, unicode, unicode] before_colon = [] after_colon = [] colon = '' found_colon = False - for i, source in enumerate(_xref_regex.split(line)): + for i, source in enumerate(_xref_regex.split(line)): # type: ignore if found_colon: after_colon.append(source) else: - if (i % 2) == 0 and ":" in source: + m = _single_colon_regex.search(source) + if (i % 2) == 0 and m: found_colon = True - before, colon, after = source.partition(":") - before_colon.append(before) - after_colon.append(after) + colon = source[m.start(): m.end()] + before_colon.append(source[:m.start()]) + after_colon.append(source[m.end():]) else: before_colon.append(source) @@ -643,7 +788,20 @@ class GoogleDocstring(object): colon, "".join(after_colon).strip()) + def _qualify_name(self, attr_name, klass): + # type: (unicode, Type) -> unicode + if klass and '.' not in attr_name: + if attr_name.startswith('~'): + attr_name = attr_name[1:] + try: + q = klass.__qualname__ + except AttributeError: + q = klass.__name__ + return '~%s.%s' % (q, attr_name) + return attr_name + def _strip_empty(self, lines): + # type: (List[unicode]) -> List[unicode] if lines: start = -1 for i, line in enumerate(lines): @@ -668,36 +826,34 @@ class NumpyDocstring(GoogleDocstring): Parameters ---------- - docstring : str or List[str] + docstring : :obj:`str` or :obj:`list` of :obj:`str` The docstring to parse, given either as a string or split into individual lines. - config : Optional[sphinxcontrib.napoleon.Config or sphinx.config.Config] + config: :obj:`sphinxcontrib.napoleon.Config` or :obj:`sphinx.config.Config` The configuration settings to use. If not given, defaults to the config object on `app`; or if `app` is not given defaults to the - a new `sphinxcontrib.napoleon.Config` object. + a new :class:`sphinxcontrib.napoleon.Config` object. - See Also - -------- - :class:`sphinxcontrib.napoleon.Config` Other Parameters ---------------- - app : Optional[sphinx.application.Sphinx] + app : :class:`sphinx.application.Sphinx`, optional Application object representing the Sphinx process. - what : Optional[str] + what : :obj:`str`, optional A string specifying the type of the object to which the docstring belongs. Valid values: "module", "class", "exception", "function", "method", "attribute". - name : Optional[str] + name : :obj:`str`, optional The fully qualified name of the object. obj : module, class, exception, function, method, or attribute The object to which the docstring belongs. - options : Optional[sphinx.ext.autodoc.Options] + options : :class:`sphinx.ext.autodoc.Options`, optional The options given to the directive: an object with attributes inherited_members, undoc_members, show_inheritance and noindex that are True if the flag option of same name was given to the auto directive. + Example ------- >>> from sphinxcontrib.napoleon import Config @@ -754,40 +910,40 @@ class NumpyDocstring(GoogleDocstring): Returns ------- - List[str] + list(str) The lines of the docstring in a list. """ def __init__(self, docstring, config=None, app=None, what='', name='', obj=None, options=None): + # type: (Union[unicode, List[unicode]], SphinxConfig, Sphinx, unicode, unicode, Any, Any) -> None # NOQA self._directive_sections = ['.. index::'] super(NumpyDocstring, self).__init__(docstring, config, app, what, name, obj, options) def _consume_field(self, parse_type=True, prefer_type=False): + # type: (bool, bool) -> Tuple[unicode, unicode, List[unicode]] line = next(self._line_iter) if parse_type: _name, _, _type = self._partition_field_on_colon(line) else: _name, _type = line, '' _name, _type = _name.strip(), _type.strip() + _name = self._escape_args_and_kwargs(_name) + if prefer_type and not _type: _type, _name = _name, _type - - if _name[:2] == '**': - _name = r'\*\*'+_name[2:] - elif _name[:1] == '*': - _name = r'\*'+_name[1:] - - indent = self._get_indent(line) - _desc = self._dedent(self._consume_indented_block(indent + 1)) + indent = self._get_indent(line) + 1 + _desc = self._dedent(self._consume_indented_block(indent)) _desc = self.__class__(_desc, self._config).lines() return _name, _type, _desc def _consume_returns_section(self): + # type: () -> List[Tuple[unicode, unicode, List[unicode]]] return self._consume_fields(prefer_type=True) def _consume_section_header(self): + # type: () -> unicode section = next(self._line_iter) if not _directive_regex.match(section): # Consume the header underline @@ -795,6 +951,7 @@ class NumpyDocstring(GoogleDocstring): return section def _is_section_break(self): + # type: () -> bool line1, line2 = self._line_iter.peek(2) return (not self._line_iter.has_next() or self._is_section_header() or @@ -804,10 +961,11 @@ class NumpyDocstring(GoogleDocstring): not self._is_indented(line1, self._section_indent))) def _is_section_header(self): + # type: () -> bool section, underline = self._line_iter.peek(2) section = section.lower() if section in self._sections and isinstance(underline, string_types): - return bool(_numpy_section_regex.match(underline)) + return bool(_numpy_section_regex.match(underline)) # type: ignore elif self._directive_sections: if _directive_regex.match(section): for directive_section in self._directive_sections: @@ -815,10 +973,8 @@ class NumpyDocstring(GoogleDocstring): return True return False - _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" - r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) - def _parse_see_also_section(self, section): + # type: (unicode) -> List[unicode] lines = self._consume_to_next_section() try: return self._parse_numpydoc_see_also_section(lines) @@ -826,6 +982,7 @@ class NumpyDocstring(GoogleDocstring): return self._format_admonition('seealso', lines) def _parse_numpydoc_see_also_section(self, content): + # type: (List[unicode]) -> List[unicode] """ Derived from the NumpyDoc implementation of _parse_see_also. @@ -840,8 +997,9 @@ class NumpyDocstring(GoogleDocstring): items = [] def parse_item_name(text): + # type: (unicode) -> Tuple[unicode, unicode] """Match ':role:`name`' or 'name'""" - m = self._name_rgx.match(text) + m = self._name_rgx.match(text) # type: ignore if m: g = m.groups() if g[1] is None: @@ -851,6 +1009,7 @@ class NumpyDocstring(GoogleDocstring): raise ValueError("%s is not a item name" % text) def push_item(name, rest): + # type: (unicode, List[unicode]) -> None if not name: return name, role = parse_item_name(name) @@ -858,13 +1017,13 @@ class NumpyDocstring(GoogleDocstring): del rest[:] current_func = None - rest = [] + rest = [] # type: List[unicode] for line in content: if not line.strip(): continue - m = self._name_rgx.match(line) + m = self._name_rgx.match(line) # type: ignore if m and line[m.end():].strip().startswith(':'): push_item(current_func, rest) current_func, line = line[:m.end()], line[m.end():] @@ -904,12 +1063,12 @@ class NumpyDocstring(GoogleDocstring): 'const': 'const', 'attribute': 'attr', 'attr': 'attr' - } + } # type: Dict[unicode, unicode] if self._what is None: - func_role = 'obj' + func_role = 'obj' # type: unicode else: func_role = roles.get(self._what, '') - lines = [] + lines = [] # type: List[unicode] last_had_desc = True for func, desc, role in items: if role: diff --git a/python/python-psi-impl/src/com/jetbrains/python/documentation/docstrings/SectionBasedDocString.java b/python/python-psi-impl/src/com/jetbrains/python/documentation/docstrings/SectionBasedDocString.java index d26bc0fdafb8..b0b74d6a8fae 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/documentation/docstrings/SectionBasedDocString.java +++ b/python/python-psi-impl/src/com/jetbrains/python/documentation/docstrings/SectionBasedDocString.java @@ -148,7 +148,7 @@ public abstract class SectionBasedDocString extends DocStringLineParser implemen /** * Used to parse e.g. optional function signature at the beginning of NumPy-style docstring * - * @return first line from which to star parsing remaining sections + * @return first line from which to start parsing remaining sections */ protected int parseHeader(int startLine) { return startLine;