sphinx-autoapi/autoapi/mappers/python.py

import re
import sys
import os
import textwrap
import ast
import tokenize as tk
import collections

import astroid
import sphinx
import sphinx.util.docstrings

from .base import PythonMapperBase, SphinxMapperBase
from . import astroid_utils
from ..utils import slugify

if sys.version_info < (3,):
    from itertools import izip_longest as zip_longest
else:
    from itertools import zip_longest


class PythonSphinxMapper(SphinxMapperBase):

    """Auto API domain handler for Python

    Parses directly from Python files.

    :param app: Sphinx application passed in as part of the extension
    """

    def load(self, patterns, dirs, ignore=None):
        """Load objects from the filesystem into the ``paths`` dictionary

        Also include an attribute on the object, ``relative_path`` which is the
        shortened, relative path the package/module
        """
        for dir_ in dirs:
            for path in self.find_files(patterns=patterns, dirs=[dir_], ignore=ignore):
                data = self.read_file(path=path)
                if data:
                    data['relative_path'] = os.path.relpath(path, dir_)
                    self.paths[path] = data

    def read_file(self, path, **kwargs):
        """Read file input into memory, returning deserialized objects

        :param path: Path of file to read
        """
        try:
            parsed_data = Parser().parse_file(path)
            return parsed_data
        except (IOError, TypeError, ImportError):
            self.app.warn('Error reading file: {0}'.format(path))
        return None

    def map(self, options=None):
        super(PythonSphinxMapper, self).map(options)

        parents = {obj.name: obj for obj in self.objects.values()}
        for obj in self.objects.values():
            parent_name = obj.name.rsplit('.', 1)[0]
            if parent_name in parents and parent_name != obj.name:
                parent = parents[parent_name]
                attr = 'sub{}s'.format(obj.type)
                getattr(parent, attr).append(obj)

        for obj in self.objects.values():
            obj.submodules.sort()
            obj.subpackages.sort()

    def create_class(self, data, options=None, **kwargs):
        """Create a class from the passed in data

        :param data: dictionary data of parser output
        """
        obj_map = dict((cls.type, cls) for cls
                       in [PythonClass, PythonFunction, PythonModule,
                           PythonMethod, PythonPackage, PythonAttribute,
                           PythonData])
        try:
            cls = obj_map[data['type']]
        except KeyError:
            self.app.warn("Unknown type: %s" % data['type'])
        else:
            obj = cls(data, jinja_env=self.jinja_env,
                      options=self.app.config.autoapi_options, **kwargs)

            lines = sphinx.util.docstrings.prepare_docstring(obj.docstring)
            try:
                if lines:
                    self.app.emit(
                        'autodoc-process-docstring',
                        cls.type,
                        obj.name,
                        None,  # object
                        None,  # options
                        lines,
                    )
            except KeyError:
                if (sphinx.version_info >= (1, 6)
                        and 'autodoc-process-docstring' in self.app.events.events):
                    raise
            else:
                obj.docstring = '\n'.join(lines)

            for child_data in data.get('children', []):
                for child_obj in self.create_class(child_data, options=options,
                                                   **kwargs):
                    obj.children.append(child_obj)
            yield obj

    def _output_top_rst(self, root):
        # Render Top Index
        top_level_index = os.path.join(root, 'index.rst')
        pages = [obj for obj in self.objects.values() if '.' not in obj.name]
        with open(top_level_index, 'w+') as top_level_file:
            content = self.jinja_env.get_template('index.rst')
            top_level_file.write(content.render(pages=pages))


class PythonPythonMapper(PythonMapperBase):

    language = 'python'
    is_callable = False

    def __init__(self, obj, **kwargs):
        super(PythonPythonMapper, self).__init__(obj, **kwargs)

        self.name = obj['name']
        self.id = slugify(self.name)

        # Optional
        self.children = []
        self.args = obj.get('args')
        self.docstring = obj['doc']

        # For later
        self.item_map = collections.defaultdict(list)

    @property
    def args(self):
        return self._args

    @args.setter
    def args(self, value):
        self._args = value

    @property
    def is_undoc_member(self):
        return not bool(self.docstring)

    @property
    def is_private_member(self):
        return (
            self.short_name.startswith('_')
            and not self.short_name.endswith('__')
        )

    @property
    def is_special_member(self):
        return (
            self.short_name.startswith('__')
            and self.short_name.endswith('__')
        )

    @property
    def display(self):
        if self.is_undoc_member and 'undoc-members' not in self.options:
            return False
        if self.is_private_member and 'private-members' not in self.options:
            return False
        if self.is_special_member and 'special-members' not in self.options:
            return False
        return True

    @property
    def summary(self):
        for line in self.docstring.splitlines():
            line = line.strip()
            if line:
                return line

        return ''

    def _children_of_type(self, type_):
        return list(child for child in self.children if child.type == type_)


class PythonFunction(PythonPythonMapper):
    type = 'function'
    is_callable = True
    ref_directive = 'func'


class PythonMethod(PythonPythonMapper):
    type = 'method'
    is_callable = True
    ref_directive = 'meth'


class PythonData(PythonPythonMapper):
    """Global, module level data."""
    type = 'data'

    def __init__(self, obj, **kwargs):
        super(PythonData, self).__init__(obj, **kwargs)

        self.value = obj.get('value')


class PythonAttribute(PythonData):
    """An object/class level attribute."""
    type = 'attribute'


class TopLevelPythonPythonMapper(PythonPythonMapper):
    top_level_object = True
    ref_directive = 'mod'

    def __init__(self, obj, **kwargs):
        super(TopLevelPythonPythonMapper, self).__init__(obj, **kwargs)

        self._resolve_name()

        self.subpackages = []
        self.submodules = []

    @property
    def functions(self):
        return self._children_of_type('function')

    @property
    def classes(self):
        return self._children_of_type('class')


class PythonModule(TopLevelPythonPythonMapper):
    type = 'module'

    def _resolve_name(self):
        name = self.obj['relative_path']
        name = name.replace('/', '.')
        ext = '.py'
        if name.endswith(ext):
            name = name[:-len(ext)]

        self.name = name


class PythonPackage(TopLevelPythonPythonMapper):
    type = 'package'

    def _resolve_name(self):
        name = self.obj['relative_path']

        exts = ['/__init__.py', '.py']
        for ext in exts:
            if name.endswith(ext):
                name = name[:-len(ext)]
                name = name.replace('/', '.')

        self.name = name


class PythonClass(PythonPythonMapper):
    type = 'class'

    def __init__(self, obj, **kwargs):
        super(PythonClass, self).__init__(obj, **kwargs)

        self.bases = obj['bases']

    @PythonPythonMapper.args.getter
    def args(self):
        args = self._args

        for child in self.children:
            if child.short_name == '__init__':
                args = child.args
                break

        if args.startswith('self'):
            args = args[4:].lstrip(',').lstrip()

        return args

    @property
    def methods(self):
        return self._children_of_type('method')

    @property
    def attributes(self):
        return self._children_of_type('attribute')


class Parser(object):
    def parse_file(self, file_path):
        directory, filename = os.path.split(file_path)
        module_part = os.path.splitext(filename)[0]
        module_parts = collections.deque([module_part])
        while os.path.isfile(os.path.join(directory, '__init__.py')):
            directory, module_part = os.path.split(directory)
            if module_part:
                module_parts.appendleft(module_part)

        module_name = '.'.join(module_parts)
        node = astroid.MANAGER.ast_from_file(file_path, module_name)
        return self.parse(node)

    def parse_assign(self, node):
        doc = ''
        doc_node = node.next_sibling()
        if (isinstance(doc_node, astroid.nodes.Expr)
                and isinstance(doc_node.value, astroid.nodes.Const)):
            doc = doc_node.value.value

        type_ = 'data'
        if (isinstance(node.scope(), astroid.nodes.ClassDef)
                or astroid_utils.is_constructor(node.scope())):
            type_ = 'attribute'

        assign_value = astroid_utils.get_assign_value(node)
        if not assign_value:
            return []

        target, value = assign_value
        data = {
            'type': type_,
            'name': target,
            'doc': doc,
            'value': value,
            'from_line_no': node.fromlineno,
            'to_line_no': node.tolineno,
        }

        return [data]

    def parse_classdef(self, node, data=None):
        args = ''
        try:
            constructor = node.lookup('__init__')[1]
        except IndexError:
            pass
        else:
            if isinstance(constructor, astroid.nodes.FunctionDef):
                args = constructor.args.as_string()

        basenames = list(astroid_utils.get_full_basenames(node.bases, node.basenames))

        data = {
            'type': 'class',
            'name': node.name,
            'args': args,
            'bases': basenames,
            'doc': node.doc or '',
            'from_line_no': node.fromlineno,
            'to_line_no': node.tolineno,
            'children': [],
        }

        for child in node.get_children():
            child_data = self.parse(child)
            if child_data:
                data['children'].extend(child_data)

        return [data]

    def _parse_property(self, node):
        data = {
            'type': 'attribute',
            'name': node.name,
            'doc': node.doc or '',
            'from_line_no': node.fromlineno,
            'to_line_no': node.tolineno,
        }

        return [data]

    def parse_functiondef(self, node):
        if astroid_utils.is_decorated_with_property(node):
            return self._parse_property(node)
        elif astroid_utils.is_decorated_with_property_setter(node):
            return []

        type_ = 'function'
        if isinstance(node.parent.scope(), astroid.nodes.ClassDef):
            type_ = 'method'

        data = {
            'type': type_,
            'name': node.name,
            'args': node.args.as_string(),
            'doc': node.doc or '',
            'from_line_no': node.fromlineno,
            'to_line_no': node.tolineno,
        }

        result = [data]

        if node.name == '__init__':
            for child in node.get_children():
                if isinstance(child, astroid.Assign):
                    child_data = self.parse_assign(child)
                    result.extend(child_data)

        return result

    def parse_module(self, node):
        type_ = 'module'
        if node.path.endswith('__init__.py'):
            type_ = 'package'

        data = {
            'type': type_,
            'name': node.name,
            'doc': node.doc or '',
            'children': [],
            'file_path': node.path,
        }

        for child in node.get_children():
            child_data = self.parse(child)
            if child_data:
                data['children'].extend(child_data)

        return data

    def parse(self, node):
        data = {}

        node_type = node.__class__.__name__.lower()
        parse_func = getattr(self, 'parse_' + node_type, None)
        if parse_func:
            data = parse_func(node)
        else:
            for child in node.get_children():
                data = self.parse(child)
                if data:
                    break

        return data