Refactor mapper classes into their bases

This separate used to exist to support parsing multiple languages,
which we no longer do.

autoapi/_astroid_utils.py

@@ -4,11 +4,7 @@ import re
 
 import astroid
 import astroid.nodes
-
 import astroid.nodes.node_classes
-import sphinx.util.logging
-
-_LOGGER = sphinx.util.logging.getLogger(__name__)
 
 
 def resolve_import_alias(name, import_names):

autoapi/_mapper.py

@@ -1,20 +1,23 @@
 import collections
 import copy
+import fnmatch
 import operator
 import os
 import re
 
+from jinja2 import Environment, FileSystemLoader
+import sphinx
 import sphinx.environment
 from sphinx.errors import ExtensionError
 import sphinx.util
+import sphinx.util.logging
 from sphinx.util.console import colorize
 from sphinx.util.display import status_iterator
 import sphinx.util.docstrings
-import sphinx.util.logging
+from sphinx.util.osutil import ensuredir
 
-from ..base import SphinxMapperBase
+from ._parser import Parser
-from .parser import Parser
+from ._objects import (
-from .objects import (
     PythonClass,
     PythonFunction,
     PythonModule,
@@ -26,6 +29,7 @@ from .objects import (
     PythonException,
     TopLevelPythonPythonMapper,
 )
+from .settings import OWN_PAGE_LEVELS, TEMPLATE_DIR
 
 LOGGER = sphinx.util.logging.getLogger(__name__)
 
@@ -219,13 +223,11 @@ def _link_objs(value):
     return result[:-2]
 
 
-class PythonSphinxMapper(SphinxMapperBase):
+class Mapper:
-    """AutoAPI domain handler for Python
+    """Base class for mapping `PythonMapperBase` objects to Sphinx.
-
-    Parses directly from Python files.
 
     Args:
-        app: Sphinx application passed in as part of the extension
+        app: Sphinx application instance
     """
 
     _OBJ_MAP = {
@@ -244,13 +246,142 @@ class PythonSphinxMapper(SphinxMapperBase):
     }
 
     def __init__(self, app, template_dir=None, dir_root=None, url_root=None):
-        super().__init__(app, template_dir, dir_root, url_root)
+        self.app = app
+
+        template_paths = [TEMPLATE_DIR]
+
+        if template_dir:
+            # Put at the front so it's loaded first
+            template_paths.insert(0, template_dir)
+
+        self.jinja_env = Environment(
+            loader=FileSystemLoader(template_paths),
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+
+        def _wrapped_prepare(value):
+            return value
+
+        self.jinja_env.filters["prepare_docstring"] = _wrapped_prepare
+        if self.app.config.autoapi_prepare_jinja_env:
+            self.app.config.autoapi_prepare_jinja_env(self.jinja_env)
+
+        own_page_level = self.app.config.autoapi_own_page_level
+        desired_page_level = OWN_PAGE_LEVELS.index(own_page_level)
+        self.own_page_types = set(OWN_PAGE_LEVELS[: desired_page_level + 1])
+
+        self.dir_root = dir_root
+        self.url_root = url_root
+
+        # Mapping of {filepath -> raw data}
+        self.paths = collections.OrderedDict()
+        # Mapping of {object id -> Python Object}
+        self.objects_to_render = collections.OrderedDict()
+        # Mapping of {object id -> Python Object}
+        self.all_objects = collections.OrderedDict()
+        # Mapping of {namespace id -> Python Object}
+        self.namespaces = collections.OrderedDict()
 
         self.jinja_env.filters["link_objs"] = _link_objs
         self._use_implicit_namespace = (
             self.app.config.autoapi_python_use_implicit_namespaces
         )
 
+    @staticmethod
+    def find_files(patterns, dirs, ignore):
+        if not ignore:
+            ignore = []
+
+        pattern_regexes = []
+        for pattern in patterns:
+            regex = re.compile(fnmatch.translate(pattern).replace(".*", "(.*)"))
+            pattern_regexes.append((pattern, regex))
+
+        for _dir in dirs:
+            for root, _, filenames in os.walk(_dir):
+                seen = set()
+                for pattern, pattern_re in pattern_regexes:
+                    for filename in fnmatch.filter(filenames, pattern):
+                        skip = False
+
+                        match = re.match(pattern_re, filename)
+                        norm_name = match.groups()
+                        if norm_name in seen:
+                            continue
+
+                        # Skip ignored files
+                        for ignore_pattern in ignore:
+                            if fnmatch.fnmatch(
+                                os.path.join(root, filename), ignore_pattern
+                            ):
+                                LOGGER.info(
+                                    colorize("bold", "[AutoAPI] ")
+                                    + colorize(
+                                        "darkgreen", f"Ignoring {root}/{filename}"
+                                    )
+                                )
+                                skip = True
+
+                        if skip:
+                            continue
+
+                        # Make sure the path is full
+                        if not os.path.isabs(filename):
+                            filename = os.path.join(root, filename)
+
+                        yield filename
+                        seen.add(norm_name)
+
+    def add_object(self, obj):
+        """Add object to local and app environment storage
+
+        Args:
+            obj: Instance of a AutoAPI object
+        """
+        display = obj.display
+        if display and obj.type in self.own_page_types:
+            self.objects_to_render[obj.id] = obj
+
+        self.all_objects[obj.id] = obj
+        child_stack = list(obj.children)
+        while child_stack:
+            child = child_stack.pop()
+            self.all_objects[child.id] = child
+            if display and child.type in self.own_page_types:
+                self.objects_to_render[child.id] = child
+            child_stack.extend(getattr(child, "children", ()))
+
+    def output_rst(self, source_suffix):
+        for _, obj in status_iterator(
+            self.objects_to_render.items(),
+            colorize("bold", "[AutoAPI] ") + "Rendering Data... ",
+            length=len(self.objects_to_render),
+            verbosity=1,
+            stringify_func=(lambda x: x[0]),
+        ):
+            rst = obj.render(is_own_page=True)
+            if not rst:
+                continue
+
+            output_dir = obj.output_dir(self.dir_root)
+            ensuredir(output_dir)
+            output_path = output_dir / obj.output_filename()
+            path = f"{output_path}{source_suffix}"
+            with open(path, "wb+") as detail_file:
+                detail_file.write(rst.encode("utf-8"))
+
+        if self.app.config.autoapi_add_toctree_entry:
+            self._output_top_rst()
+
+    def _output_top_rst(self):
+        # Render Top Index
+        top_level_index = os.path.join(self.dir_root, "index.rst")
+        pages = [obj for obj in self.objects_to_render.values() if obj.display]
+        with open(top_level_index, "wb") as top_level_file:
+            content = self.jinja_env.get_template("index.rst")
+            top_level_file.write(content.render(pages=pages).encode("utf-8"))
+
     def _need_to_load(self, files):
         last_files = getattr(self.app.env, "autoapi_source_files", [])
         self.app.env.autoapi_source_files = files
@@ -361,7 +492,14 @@ class PythonSphinxMapper(SphinxMapperBase):
         self._hide_yo_kids()
         self.app.env.autoapi_annotations = {}
 
-        super().map(options)
+        for _, data in status_iterator(
+            self.paths.items(),
+            colorize("bold", "[AutoAPI] ") + "Mapping Data... ",
+            length=len(self.paths),
+            stringify_func=(lambda x: x[0]),
+        ):
+            for obj in self.create_class(data, options=options):
+                self.add_object(obj)
 
         top_level_objects = {
             obj.id: obj
@@ -383,7 +521,7 @@ class PythonSphinxMapper(SphinxMapperBase):
         self.app.env.autoapi_objects = self.objects_to_render
         self.app.env.autoapi_all_objects = self.all_objects
 
-    def create_class(self, data, options=None, **kwargs):
+    def create_class(self, data, options=None):
         """Create a class from the passed in data
 
         Args:
@@ -402,13 +540,10 @@ class PythonSphinxMapper(SphinxMapperBase):
                 jinja_env=self.jinja_env,
                 app=self.app,
                 url_root=self.url_root,
-                **kwargs,
             )
 
             for child_data in data.get("children", []):
-                for child_obj in self.create_class(
+                for child_obj in self.create_class(child_data, options=options):
-                    child_data, options=options, **kwargs
-                ):
                     obj.children.append(child_obj)
 
             # Some objects require children to establish their docstring

autoapi/_objects.py

@@ -1,11 +1,14 @@
+from __future__ import annotations
+
 import functools
 import pathlib
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
+import sphinx
+import sphinx.util
 import sphinx.util.logging
 
-from ..base import PythonMapperBase
+from .settings import OWN_PAGE_LEVELS
-
 
 LOGGER = sphinx.util.logging.getLogger(__name__)
 
@@ -27,105 +30,176 @@ def _format_args(args_info, include_annotations=True, ignore_self=None):
     return ", ".join(result)
 
 
-class PythonPythonMapper(PythonMapperBase):
+class PythonObject:
-    """A base class for all types of representations of Python objects.
+    """A class representing an entity from the parsed source code.
 
-    Attributes:
+    This class turns the dictionaries output by the parser into an object.
-        name (str): The name given to this object.
+
-        id (str): A unique identifier for this object.
+    Args:
-        children (list(PythonPythonMapper)): The members of this object.
+        obj: JSON object representing this object
+        jinja_env: A template environment for rendering this object
     """
 
-    language = "python"
-    is_callable = False
     member_order = 0
+    """The ordering of objects when doing "groupwise" sorting."""
     type: str
 
-    def __init__(self, obj, class_content="class", **kwargs) -> None:
+    def __init__(
-        super().__init__(obj, **kwargs)
+        self, obj, jinja_env, app, url_root, options=None, class_content="class"
+    ):
+        self.app = app
+        self.obj = obj
+        self.options = options
+        self.jinja_env = jinja_env
+        self.url_root = url_root
+
+        self.name: str = obj["name"]
+        """The name of the object, as named in the parsed source code.
 
-        self.name = obj["name"]
+        This name will have no periods in it.
-        self.qual_name = obj["qual_name"]
+        """
-        self.id = obj.get("full_name", self.name)
+        self.qual_name: str = obj["qual_name"]
+        """The qualified name for this object."""
+        self.id: str = obj.get("full_name", self.name)
+        """A globally unique identifier for this object.
+
+        This is the same as the fully qualified name of the object.
+        """
 
-        # Optional
+        self.children: List[PythonObject] = []
-        self.children: List[PythonPythonMapper] = []
+        """The members of this object.
-        self._docstring = obj["doc"]
-        self._docstring_resolved = False
-        self.imported = "original_path" in obj
-        self.inherited = obj.get("inherited", False)
-        """Whether this was inherited from an ancestor of the parent class.
 
-        :type: bool
+        For example, the classes and functions defined in the parent module.
         """
+        self._docstring: str = obj["doc"]
+        self.imported: bool = "original_path" in obj
+        """Whether this object was imported from another module."""
+        self.inherited: bool = obj.get("inherited", False)
+        """Whether this was inherited from an ancestor of the parent class."""
         self._hide = obj.get("hide", False)
 
         # For later
         self._class_content = class_content
-
         self._display_cache: Optional[bool] = None
 
+    def __getstate__(self):
+        """Obtains serialisable data for pickling."""
+        __dict__ = self.__dict__.copy()
+        __dict__.update(app=None, jinja_env=None)  # clear unpickable attributes
+        return __dict__
+
+    def render(self, **kwargs):
+        LOGGER.log("VERBOSE", "Rendering %s", self.id)
+
+        template = self.jinja_env.get_template(f"python/{self.type}.rst")
+
+        ctx = {}
+        ctx.update(**self.get_context_data())
+        ctx.update(**kwargs)
+        return template.render(**ctx)
+
+    @property
+    def rendered(self):
+        """Shortcut to render an object in templates."""
+        return self.render()
+
+    def get_context_data(self):
+        own_page_level = self.app.config.autoapi_own_page_level
+        desired_page_level = OWN_PAGE_LEVELS.index(own_page_level)
+        own_page_types = set(OWN_PAGE_LEVELS[: desired_page_level + 1])
+
+        return {
+            "autoapi_options": self.app.config.autoapi_options,
+            "include_summaries": self.app.config.autoapi_include_summaries,
+            "obj": self,
+            "own_page_types": own_page_types,
+            "sphinx_version": sphinx.version_info,
+        }
+
+    def __lt__(self, other):
+        """Object sorting comparison"""
+        if not isinstance(other, PythonObject):
+            return NotImplemented
+
+        return self.id < other.id
+
+    def __str__(self) -> str:
+        return f"<{self.__class__.__name__} {self.id}>"
+
+    @property
+    def short_name(self) -> str:
+        """Shorten name property"""
+        return self.name.split(".")[-1]
+
+    def output_dir(self, root):
+        """The directory to render this object."""
+        module = self.id[: -(len("." + self.qual_name))]
+        parts = [root] + module.split(".")
+        return pathlib.PurePosixPath(*parts)
+
+    def output_filename(self) -> str:
+        """The name of the file to render into, without a file suffix."""
+        filename = self.qual_name
+        if filename == "index":
+            filename = ".index"
+
+        return filename
+
+    @property
+    def include_path(self) -> str:
+        """Return 'absolute' path without regarding OS path separator
+
+        This is used in ``toctree`` directives, as Sphinx always expects Unix
+        path separators
+        """
+        return str(self.output_dir(self.url_root) / self.output_filename())
+
     @property
-    def docstring(self):
+    def docstring(self) -> str:
         """The docstring for this object.
 
         If a docstring did not exist on the object,
         this will be the empty string.
 
-        For classes this will also depend on the
+        For classes, this will also depend on the
         :confval:`autoapi_python_class_content` option.
-
-        :type: str
         """
         return self._docstring
 
     @docstring.setter
-    def docstring(self, value):
+    def docstring(self, value: str) -> None:
         self._docstring = value
         self._docstring_resolved = True
 
     @property
-    def is_top_level_object(self):
+    def is_top_level_object(self) -> bool:
         """Whether this object is at the very top level (True) or not (False).
 
         This will be False for subpackages and submodules.
-
-        :type: bool
         """
         return "." not in self.id
 
     @property
-    def is_undoc_member(self):
+    def is_undoc_member(self) -> bool:
-        """Whether this object has a docstring (False) or not (True).
+        """Whether this object has a docstring (False) or not (True)."""
-
-        :type: bool
-        """
         return not bool(self.docstring)
 
     @property
-    def is_private_member(self):
+    def is_private_member(self) -> bool:
-        """Whether this object is private (True) or not (False).
+        """Whether this object is private (True) or not (False)."""
-
-        :type: bool
-        """
         return self.short_name.startswith("_") and not self.short_name.endswith("__")
 
     @property
-    def is_special_member(self):
+    def is_special_member(self) -> bool:
-        """Whether this object is a special member (True) or not (False).
+        """Whether this object is a special member (True) or not (False)."""
-
-        :type: bool
-        """
         return self.short_name.startswith("__") and self.short_name.endswith("__")
 
     @property
-    def display(self):
+    def display(self) -> bool:
         """Whether this object should be displayed in documentation.
 
         This attribute depends on the configuration options given in
         :confval:`autoapi_options` and the result of :event:`autoapi-skip-member`.
-
-        :type: bool
         """
         if self._display_cache is None:
             self._display_cache = not self._ask_ignore(self._should_skip())
@@ -133,13 +207,11 @@ class PythonPythonMapper(PythonMapperBase):
         return self._display_cache
 
     @property
-    def summary(self):
+    def summary(self) -> str:
         """The summary line of the docstring.
 
         The summary line is the first non-empty line, as-per :pep:`257`.
         This will be the empty string if the object does not have a docstring.
-
-        :type: str
         """
         for line in self.docstring.splitlines():
             line = line.strip()
@@ -148,7 +220,7 @@ class PythonPythonMapper(PythonMapperBase):
 
         return ""
 
-    def _should_skip(self):  # type: () -> bool
+    def _should_skip(self) -> bool:
         skip_undoc_member = self.is_undoc_member and "undoc-members" not in self.options
         skip_private_member = (
             self.is_private_member and "private-members" not in self.options
@@ -170,61 +242,53 @@ class PythonPythonMapper(PythonMapperBase):
             or skip_inherited_member
         )
 
-    def _ask_ignore(self, skip):  # type: (bool) -> bool
+    def _ask_ignore(self, skip: bool) -> bool:
         ask_result = self.app.emit_firstresult(
             "autoapi-skip-member", self.type, self.id, self, skip, self.options
         )
 
         return ask_result if ask_result is not None else skip
 
-    def _children_of_type(self, type_):
+    def _children_of_type(self, type_: str) -> List[PythonObject]:
         return list(child for child in self.children if child.type == type_)
 
 
-class PythonFunction(PythonPythonMapper):
+class PythonFunction(PythonObject):
     """The representation of a function."""
 
     type = "function"
-    is_callable = True
     member_order = 30
 
-    def __init__(self, obj, **kwargs):
+    def __init__(self, *args, **kwargs):
-        super().__init__(obj, **kwargs)
+        super().__init__(*args, **kwargs)
 
         autodoc_typehints = getattr(self.app.config, "autodoc_typehints", "signature")
         show_annotations = autodoc_typehints != "none" and not (
-            autodoc_typehints == "description" and not obj["overloads"]
+            autodoc_typehints == "description" and not self.obj["overloads"]
         )
-        self.args = _format_args(obj["args"], show_annotations)
+        self.args: str = _format_args(self.obj["args"], show_annotations)
-        """The arguments to this object, formatted as a string.
+        """The arguments to this object, formatted as a string."""
 
-        :type: str
+        self.return_annotation: Optional[str] = (
-        """
+            self.obj["return_annotation"] if show_annotations else None
-
+        )
-        self.return_annotation = obj["return_annotation"] if show_annotations else None
         """The type annotation for the return type of this function.
 
         This will be ``None`` if an annotation
         or annotation comment was not given.
-
-        :type: str or None
         """
-        self.properties = obj["properties"]
+        self.properties: List[str] = self.obj["properties"]
         """The properties that describe what type of function this is.
 
-        Can be only be: async
+        Can be only be: async.
-
-        :type: list(str)
         """
-        self.overloads = [
+        self.overloads: List[Tuple[str, str]] = [
             (_format_args(args), return_annotation)
-            for args, return_annotation in obj["overloads"]
+            for args, return_annotation in self.obj["overloads"]
         ]
         """The overloaded signatures of this function.
 
         Each tuple is a tuple of ``(args, return_annotation)``
-
-        :type: list(tuple(str, str))
         """
 
 
@@ -232,73 +296,61 @@ class PythonMethod(PythonFunction):
     """The representation of a method."""
 
     type = "method"
-    is_callable = True
     member_order = 50
 
-    def __init__(self, obj, **kwargs):
+    def __init__(self, *args, **kwargs):
-        super().__init__(obj, **kwargs)
+        super().__init__(*args, **kwargs)
 
-        self.properties = obj["properties"]
+        self.properties: List[str] = self.obj["properties"]
         """The properties that describe what type of method this is.
 
-        Can be any of: abstractmethod, async, classmethod, property, staticmethod
+        Can be any of: abstractmethod, async, classmethod, property, staticmethod.
-
-        :type: list(str)
         """
 
-    def _should_skip(self):  # type: () -> bool
+    def _should_skip(self) -> bool:
         return super()._should_skip() or self.name in (
             "__new__",
             "__init__",
         )
 
 
-class PythonProperty(PythonPythonMapper):
+class PythonProperty(PythonObject):
     """The representation of a property on a class."""
 
     type = "property"
     member_order = 60
 
-    def __init__(self, obj, **kwargs):
+    def __init__(self, *args, **kwargs):
-        super().__init__(obj, **kwargs)
+        super().__init__(*args, **kwargs)
-
-        self.annotation = obj["return_annotation"]
-        """The type annotation of this property.
 
-        :type: str or None
+        self.annotation: Optional[str] = self.obj["return_annotation"]
-        """
+        """The type annotation of this property."""
-        self.properties = obj["properties"]
+        self.properties: List[str] = self.obj["properties"]
         """The properties that describe what type of property this is.
 
-        Can be any of: abstractmethod, classmethod
+        Can be any of: abstractmethod, classmethod.
-
-        :type: list(str)
         """
 
 
-class PythonData(PythonPythonMapper):
+class PythonData(PythonObject):
     """Global, module level data."""
 
     type = "data"
     member_order = 40
 
-    def __init__(self, obj, **kwargs):
+    def __init__(self, *args, **kwargs):
-        super().__init__(obj, **kwargs)
+        super().__init__(*args, **kwargs)
 
-        self.value = obj.get("value")
+        self.value: Optional[str] = self.obj.get("value")
         """The value of this attribute.
 
         This will be ``None`` if the value is not constant.
-
-        :type: str or None
         """
-        self.annotation = obj.get("annotation")
+        self.annotation: Optional[str] = self.obj.get("annotation")
         """The type annotation of this attribute.
 
         This will be ``None`` if an annotation
         or annotation comment was not given.
-
-        :type: str or None
         """
 
 
@@ -309,17 +361,15 @@ class PythonAttribute(PythonData):
     member_order = 60
 
 
-class TopLevelPythonPythonMapper(PythonPythonMapper):
+class TopLevelPythonPythonMapper(PythonObject):
     """A common base class for modules and packages."""
 
-    _RENDER_LOG_LEVEL = "VERBOSE"
+    def __init__(self, *args, **kwargs):
-
+        super().__init__(*args, **kwargs)
-    def __init__(self, obj, **kwargs):
-        super().__init__(obj, **kwargs)
 
         self.subpackages = []
         self.submodules = []
-        self.all = obj["all"]
+        self.all = self.obj["all"]
         """The contents of ``__all__`` if assigned to.
 
         Only constants are included.
@@ -366,27 +416,23 @@ class PythonPackage(TopLevelPythonPythonMapper):
     type = "package"
 
 
-class PythonClass(PythonPythonMapper):
+class PythonClass(PythonObject):
     """The representation of a class."""
 
     type = "class"
     member_order = 20
 
-    def __init__(self, obj, **kwargs):
+    def __init__(self, *args, **kwargs):
-        super().__init__(obj, **kwargs)
+        super().__init__(*args, **kwargs)
 
-        self.bases = obj["bases"]
+        self.bases: List[str] = self.obj["bases"]
-        """The fully qualified names of all base classes.
+        """The fully qualified names of all base classes."""
 
-        :type: list(str)
+        self._docstring_resolved: bool = False
-        """
 
     @property
-    def args(self):
+    def args(self) -> str:
-        """The arguments to this object, formatted as a string.
+        """The arguments to this object, formatted as a string."""
-
-        :type: str
-        """
         args = ""
 
         if self.constructor:
@@ -402,7 +448,7 @@ class PythonClass(PythonPythonMapper):
         return args
 
     @property
-    def overloads(self):
+    def overloads(self) -> List[Tuple[str, str]]:
         overloads = []
 
         if self.constructor:
@@ -422,8 +468,8 @@ class PythonClass(PythonPythonMapper):
         return overloads
 
     @property
-    def docstring(self):
+    def docstring(self) -> str:
-        docstring = super().docstring
+        docstring = self._docstring
 
         if not self._docstring_resolved and self._class_content in ("both", "init"):
             constructor_docstring = self.constructor_docstring
@@ -437,8 +483,9 @@ class PythonClass(PythonPythonMapper):
         return docstring
 
     @docstring.setter
-    def docstring(self, value):
+    def docstring(self, value: str) -> None:
-        super(PythonClass, self.__class__).docstring.fset(self, value)
+        self._docstring = value
+        self._docstring_resolved = True
 
     @property
     def methods(self):
@@ -466,7 +513,7 @@ class PythonClass(PythonPythonMapper):
         return None
 
     @property
-    def constructor_docstring(self):
+    def constructor_docstring(self) -> str:
         docstring = ""
 
         constructor = self.constructor

autoapi/_parser.py

@@ -6,7 +6,7 @@ import astroid
 import astroid.builder
 import sphinx.util.docstrings
 
-from . import astroid_utils
+from . import _astroid_utils
 
 
 def _prepare_docstring(doc):
@@ -80,17 +80,17 @@ class Parser:
         type_ = "data"
         if isinstance(
             node.scope(), astroid.nodes.ClassDef
-        ) or astroid_utils.is_constructor(node.scope()):
+        ) or _astroid_utils.is_constructor(node.scope()):
             type_ = "attribute"
 
-        assign_value = astroid_utils.get_assign_value(node)
+        assign_value = _astroid_utils.get_assign_value(node)
         if not assign_value:
             return []
 
         target = assign_value[0]
         value = assign_value[1]
 
-        annotation = astroid_utils.get_assign_annotation(node)
+        annotation = _astroid_utils.get_assign_annotation(node)
 
         data = {
             "type": type_,
@@ -108,10 +108,10 @@ class Parser:
 
     def parse_classdef(self, node, data=None):
         type_ = "class"
-        if astroid_utils.is_exception(node):
+        if _astroid_utils.is_exception(node):
             type_ = "exception"
 
-        basenames = list(astroid_utils.get_full_basenames(node))
+        basenames = list(_astroid_utils.get_full_basenames(node))
 
         data = {
             "type": type_,
@@ -119,7 +119,7 @@ class Parser:
             "qual_name": self._get_qual_name(node.name),
             "full_name": self._get_full_name(node.name),
             "bases": basenames,
-            "doc": _prepare_docstring(astroid_utils.get_class_docstring(node)),
+            "doc": _prepare_docstring(_astroid_utils.get_class_docstring(node)),
             "from_line_no": node.fromlineno,
             "to_line_no": node.tolineno,
             "children": [],
@@ -140,7 +140,7 @@ class Parser:
             for child in base.get_children():
                 name = getattr(child, "name", None)
                 if isinstance(child, (astroid.Assign, astroid.AnnAssign)):
-                    assign_value = astroid_utils.get_assign_value(child)
+                    assign_value = _astroid_utils.get_assign_value(child)
                     if not assign_value:
                         continue
                     name = assign_value[0]
@@ -164,7 +164,7 @@ class Parser:
         return self.parse_functiondef(node)
 
     def parse_functiondef(self, node):
-        if astroid_utils.is_decorated_with_property_setter(node):
+        if _astroid_utils.is_decorated_with_property_setter(node):
             return []
 
         type_ = "method"
@@ -175,7 +175,7 @@ class Parser:
 
             if isinstance(node, astroid.AsyncFunctionDef):
                 properties.append("async")
-        elif astroid_utils.is_decorated_with_property(node):
+        elif _astroid_utils.is_decorated_with_property(node):
             type_ = "property"
             if node.type == "classmethod":
                 properties.append(node.type)
@@ -195,13 +195,13 @@ class Parser:
             "name": node.name,
             "qual_name": self._get_qual_name(node.name),
             "full_name": self._get_full_name(node.name),
-            "args": astroid_utils.get_args_info(node.args),
+            "args": _astroid_utils.get_args_info(node.args),
-            "doc": _prepare_docstring(astroid_utils.get_func_docstring(node)),
+            "doc": _prepare_docstring(_astroid_utils.get_func_docstring(node)),
             "from_line_no": node.fromlineno,
             "to_line_no": node.tolineno,
-            "return_annotation": astroid_utils.get_return_annotation(node),
+            "return_annotation": _astroid_utils.get_return_annotation(node),
             "properties": properties,
-            "is_overload": astroid_utils.is_decorated_with_overload(node),
+            "is_overload": _astroid_utils.is_decorated_with_overload(node),
             "overloads": [],
         }
 
@@ -220,7 +220,7 @@ class Parser:
 
         for import_name, alias in node.names:
             is_wildcard = (alias or import_name) == "*"
-            original_path = astroid_utils.get_full_import_name(
+            original_path = _astroid_utils.get_full_import_name(
                 node, alias or import_name
             )
             name = original_path if is_wildcard else (alias or import_name)
@@ -259,13 +259,13 @@ class Parser:
             "children": [],
             "file_path": path,
             "encoding": node.file_encoding,
-            "all": astroid_utils.get_module_all(node),
+            "all": _astroid_utils.get_module_all(node),
         }
 
         overloads = {}
         top_name = node.name.split(".", 1)[0]
         for child in node.get_children():
-            if astroid_utils.is_local_import_from(child, top_name):
+            if _astroid_utils.is_local_import_from(child, top_name):
                 child_data = self._parse_local_import_from(child)
             else:
                 child_data = self.parse(child)

autoapi/directives.py

@@ -6,7 +6,7 @@ from docutils import nodes
 from sphinx.ext.autosummary import Autosummary, mangle_signature
 from sphinx.util.nodes import nested_parse_with_titles
 
-from .mappers.python.objects import PythonFunction
+from ._objects import PythonFunction
 
 
 class AutoapiSummary(Autosummary):

autoapi/documenters.py

@@ -2,7 +2,7 @@ import re
 
 from sphinx.ext import autodoc
 
-from .mappers.python import (
+from ._objects import (
     PythonFunction,
     PythonClass,
     PythonMethod,

autoapi/extension.py

@@ -19,7 +19,7 @@ from docutils.parsers.rst import directives
 from . import documenters
 from .directives import AutoapiSummary, NestedParse
 from .inheritance_diagrams import AutoapiInheritanceDiagram
-from .mappers import PythonSphinxMapper
+from ._mapper import Mapper
 from .settings import API_ROOT
 
 LOGGER = sphinx.util.logging.getLogger(__name__)
@@ -111,7 +111,7 @@ def run_autoapi(app):
                 "relative to where sphinx-build is run\n",
                 RemovedInAutoAPI3Warning,
             )
-    sphinx_mapper_obj = PythonSphinxMapper(
+    sphinx_mapper_obj = Mapper(
         app, template_dir=template_dir, dir_root=normalized_root, url_root=url_root
     )
 

autoapi/mappers/__init__.py

@@ -1,3 +0,0 @@
-from .python import PythonSphinxMapper
-
-__all__ = ("PythonSphinxMapper",)

autoapi/mappers/base.py

@@ -1,342 +0,0 @@
-from collections import OrderedDict, namedtuple
-import fnmatch
-import os
-import pathlib
-import re
-
-from jinja2 import Environment, FileSystemLoader, TemplateNotFound
-import sphinx
-import sphinx.util
-from sphinx.util.console import colorize
-from sphinx.util.display import status_iterator
-from sphinx.util.osutil import ensuredir
-import sphinx.util.logging
-
-from ..settings import TEMPLATE_DIR
-
-LOGGER = sphinx.util.logging.getLogger(__name__)
-_OWN_PAGE_LEVELS = [
-    "package",
-    "module",
-    "exception",
-    "class",
-    "function",
-    "method",
-    "property",
-    "data",
-    "attribute",
-]
-
-Path = namedtuple("Path", ["absolute", "relative"])
-
-
-class PythonMapperBase:
-    """Base object for JSON -> Python object mapping.
-
-    Subclasses of this object will handle their language specific JSON input,
-    and map that onto this standard Python object.
-    Subclasses may also include language-specific attributes on this object.
-
-    Args:
-        obj: JSON object representing this object
-        jinja_env: A template environment for rendering this object
-
-    Attributes:
-        id (str): A globally unique identifier for this object.
-            Generally a fully qualified name, including namespace.
-        name (str): A short "display friendly" name for this object.
-        docstring (str): The documentation for this object
-        imports (list): Imports in this object
-        children (list): Children of this object
-        parameters (list): Parameters to this object
-        methods (list): Methods on this object
-    """
-
-    language = "base"
-    type = "base"
-    _RENDER_LOG_LEVEL = "VERBOSE"
-
-    def __init__(self, obj, jinja_env, app, url_root, options=None):
-        self.app = app
-        self.obj = obj
-        self.options = options
-        self.jinja_env = jinja_env
-        self.url_root = url_root
-
-        self.name = None
-        self.qual_name = None
-        self.id = None
-
-    def __getstate__(self):
-        """Obtains serialisable data for pickling."""
-        __dict__ = self.__dict__.copy()
-        __dict__.update(app=None, jinja_env=None)  # clear unpickable attributes
-        return __dict__
-
-    def render(self, **kwargs):
-        LOGGER.log(self._RENDER_LOG_LEVEL, "Rendering %s", self.id)
-
-        ctx = {}
-        try:
-            template = self.jinja_env.get_template(f"{self.language}/{self.type}.rst")
-        except TemplateNotFound:
-            template = self.jinja_env.get_template(f"base/{self.type}.rst")
-
-        ctx.update(**self.get_context_data())
-        ctx.update(**kwargs)
-        return template.render(**ctx)
-
-    @property
-    def rendered(self):
-        """Shortcut to render an object in templates."""
-        return self.render()
-
-    def get_context_data(self):
-        own_page_level = self.app.config.autoapi_own_page_level
-        desired_page_level = _OWN_PAGE_LEVELS.index(own_page_level)
-        own_page_types = set(_OWN_PAGE_LEVELS[: desired_page_level + 1])
-
-        return {
-            "autoapi_options": self.app.config.autoapi_options,
-            "include_summaries": self.app.config.autoapi_include_summaries,
-            "obj": self,
-            "own_page_types": own_page_types,
-            "sphinx_version": sphinx.version_info,
-        }
-
-    def __lt__(self, other):
-        """Object sorting comparison"""
-        if isinstance(other, PythonMapperBase):
-            return self.id < other.id
-        return super().__lt__(other)
-
-    def __str__(self):
-        return f"<{self.__class__.__name__} {self.id}>"
-
-    @property
-    def short_name(self):
-        """Shorten name property"""
-        return self.name.split(".")[-1]
-
-    def output_dir(self, root):
-        """The directory to render this object."""
-        module = self.id[: -(len("." + self.qual_name))]
-        parts = [root] + module.split(".")
-        return pathlib.PurePosixPath(*parts)
-
-    def output_filename(self):
-        """The name of the file to render into, without a file suffix."""
-        filename = self.qual_name
-        if filename == "index":
-            filename = ".index"
-
-        return filename
-
-    @property
-    def include_path(self):
-        """Return 'absolute' path without regarding OS path separator
-
-        This is used in ``toctree`` directives, as Sphinx always expects Unix
-        path separators
-        """
-        return str(self.output_dir(self.url_root) / self.output_filename())
-
-    @property
-    def display(self):
-        """Whether to display this object or not.
-
-        :type: bool
-        """
-        return True
-
-    @property
-    def ref_type(self):
-        return self.type
-
-    @property
-    def ref_directive(self):
-        return self.type
-
-
-class SphinxMapperBase:
-    """Base class for mapping `PythonMapperBase` objects to Sphinx.
-
-    Args:
-        app: Sphinx application instance
-    """
-
-    def __init__(self, app, template_dir=None, dir_root=None, url_root=None):
-        self.app = app
-
-        template_paths = [TEMPLATE_DIR]
-
-        if template_dir:
-            # Put at the front so it's loaded first
-            template_paths.insert(0, template_dir)
-
-        self.jinja_env = Environment(
-            loader=FileSystemLoader(template_paths),
-            trim_blocks=True,
-            lstrip_blocks=True,
-        )
-
-        def _wrapped_prepare(value):
-            return value
-
-        self.jinja_env.filters["prepare_docstring"] = _wrapped_prepare
-        if self.app.config.autoapi_prepare_jinja_env:
-            self.app.config.autoapi_prepare_jinja_env(self.jinja_env)
-
-        own_page_level = self.app.config.autoapi_own_page_level
-        desired_page_level = _OWN_PAGE_LEVELS.index(own_page_level)
-        self.own_page_types = set(_OWN_PAGE_LEVELS[: desired_page_level + 1])
-
-        self.dir_root = dir_root
-        self.url_root = url_root
-
-        # Mapping of {filepath -> raw data}
-        self.paths = OrderedDict()
-        # Mapping of {object id -> Python Object}
-        self.objects_to_render = OrderedDict()
-        # Mapping of {object id -> Python Object}
-        self.all_objects = OrderedDict()
-        # Mapping of {namespace id -> Python Object}
-        self.namespaces = OrderedDict()
-
-    def load(self, patterns, dirs, ignore=None):
-        """Load objects from the filesystem into the ``paths`` dictionary."""
-        paths = list(self.find_files(patterns=patterns, dirs=dirs, ignore=ignore))
-        for path in status_iterator(
-            paths,
-            colorize("bold", "[AutoAPI] Reading files... "),
-            "darkgreen",
-            len(paths),
-        ):
-            data = self.read_file(path=path)
-            if data:
-                self.paths[path] = data
-
-        return True
-
-    @staticmethod
-    def find_files(patterns, dirs, ignore):
-        if not ignore:
-            ignore = []
-
-        pattern_regexes = []
-        for pattern in patterns:
-            regex = re.compile(fnmatch.translate(pattern).replace(".*", "(.*)"))
-            pattern_regexes.append((pattern, regex))
-
-        for _dir in dirs:
-            for root, _, filenames in os.walk(_dir):
-                seen = set()
-                for pattern, pattern_re in pattern_regexes:
-                    for filename in fnmatch.filter(filenames, pattern):
-                        skip = False
-
-                        match = re.match(pattern_re, filename)
-                        norm_name = match.groups()
-                        if norm_name in seen:
-                            continue
-
-                        # Skip ignored files
-                        for ignore_pattern in ignore:
-                            if fnmatch.fnmatch(
-                                os.path.join(root, filename), ignore_pattern
-                            ):
-                                LOGGER.info(
-                                    colorize("bold", "[AutoAPI] ")
-                                    + colorize(
-                                        "darkgreen", f"Ignoring {root}/{filename}"
-                                    )
-                                )
-                                skip = True
-
-                        if skip:
-                            continue
-
-                        # Make sure the path is full
-                        if not os.path.isabs(filename):
-                            filename = os.path.join(root, filename)
-
-                        yield filename
-                        seen.add(norm_name)
-
-    def read_file(self, path, **kwargs):
-        """Read file input into memory
-
-        Args:
-            path: Path of file to read
-        """
-        # TODO support JSON here
-        # TODO sphinx way of reporting errors in logs?
-        raise NotImplementedError
-
-    def add_object(self, obj):
-        """Add object to local and app environment storage
-
-        Args:
-            obj: Instance of a AutoAPI object
-        """
-        display = obj.display
-        if display and obj.type in self.own_page_types:
-            self.objects_to_render[obj.id] = obj
-
-        self.all_objects[obj.id] = obj
-        child_stack = list(obj.children)
-        while child_stack:
-            child = child_stack.pop()
-            self.all_objects[child.id] = child
-            if display and child.type in self.own_page_types:
-                self.objects_to_render[child.id] = child
-            child_stack.extend(getattr(child, "children", ()))
-
-    def map(self, options=None):
-        """Trigger find of serialized sources and build objects"""
-        for _, data in status_iterator(
-            self.paths.items(),
-            colorize("bold", "[AutoAPI] ") + "Mapping Data... ",
-            length=len(self.paths),
-            stringify_func=(lambda x: x[0]),
-        ):
-            for obj in self.create_class(data, options=options):
-                self.add_object(obj)
-
-    def create_class(self, data, options=None, **kwargs):
-        """Create class object.
-
-        Args:
-            data: Instance of a AutoAPI object
-        """
-        raise NotImplementedError
-
-    def output_rst(self, source_suffix):
-        for _, obj in status_iterator(
-            self.objects_to_render.items(),
-            colorize("bold", "[AutoAPI] ") + "Rendering Data... ",
-            length=len(self.objects_to_render),
-            verbosity=1,
-            stringify_func=(lambda x: x[0]),
-        ):
-            rst = obj.render(is_own_page=True)
-            if not rst:
-                continue
-
-            output_dir = obj.output_dir(self.dir_root)
-            ensuredir(output_dir)
-            output_path = output_dir / obj.output_filename()
-            path = f"{output_path}{source_suffix}"
-            with open(path, "wb+") as detail_file:
-                detail_file.write(rst.encode("utf-8"))
-
-        if self.app.config.autoapi_add_toctree_entry:
-            self._output_top_rst()
-
-    def _output_top_rst(self):
-        # Render Top Index
-        top_level_index = os.path.join(self.dir_root, "index.rst")
-        pages = [obj for obj in self.objects_to_render.values() if obj.display]
-        with open(top_level_index, "wb") as top_level_file:
-            content = self.jinja_env.get_template("index.rst")
-            top_level_file.write(content.render(pages=pages).encode("utf-8"))

autoapi/mappers/python/__init__.py

@@ -1,25 +0,0 @@
-from .mapper import PythonSphinxMapper
-from .objects import (
-    PythonAttribute,
-    PythonClass,
-    PythonData,
-    PythonException,
-    PythonFunction,
-    PythonMethod,
-    PythonModule,
-    PythonPackage,
-    PythonProperty,
-)
-
-__all__ = (
-    "PythonAttribute",
-    "PythonClass",
-    "PythonData",
-    "PythonException",
-    "PythonFunction",
-    "PythonMethod",
-    "PythonModule",
-    "PythonPackage",
-    "PythonProperty",
-    "PythonSphinxMapper",
-)

autoapi/settings.py

@@ -10,3 +10,14 @@ SITE_ROOT = os.path.dirname(os.path.realpath(__file__))
 TEMPLATE_DIR = os.path.join(SITE_ROOT, "templates")
 
 API_ROOT = "autoapi"
+OWN_PAGE_LEVELS = [
+    "package",
+    "module",
+    "exception",
+    "class",
+    "function",
+    "method",
+    "property",
+    "data",
+    "attribute",
+]

autoapi/templates/base/base.rst

@@ -1,7 +0,0 @@
-.. {{ obj.type }}:: {{ obj.name }}
-
-	{% if summary %}
-
-	{{ obj.summary }}
-
-	{% endif %}

docs/changes/+cf0a1267.misc

@@ -0,0 +1 @@
+Refactor mapper classes into their bases

docs/reference/directives.rst

@@ -55,7 +55,7 @@ Inheritance Diagrams
 
     For example:
 
-    .. autoapi-inheritance-diagram:: autoapi.mappers.python.objects.PythonModule autoapi.mappers.python.objects.PythonPackage
+    .. autoapi-inheritance-diagram:: autoapi._objects.PythonModule autoapi._objects.PythonPackage
         :parts: 1
 
     :mod:`sphinx.ext.inheritance_diagram` makes use of the

tests/python/test_parser.py

@@ -1,15 +1,8 @@
-# coding=utf8
-
 """Test Python parser"""
 
-from io import StringIO
-import sys
-from textwrap import dedent
-
 import astroid
-import pytest
 
-from autoapi.mappers.python.parser import Parser
+from autoapi._parser import Parser
 
 
 class TestPythonParser:

tests/python/test_pyintegration.py

@@ -5,7 +5,7 @@ import sys
 from unittest.mock import Mock, call
 
 import autoapi.settings
-from autoapi.mappers.python import (
+from autoapi._objects import (
     PythonClass,
     PythonData,
     PythonFunction,

tests/test_astroid_utils.py

@@ -1,7 +1,8 @@
 import sys
 
 import astroid
-from autoapi.mappers.python import astroid_utils, objects
+from autoapi import _astroid_utils
+from autoapi import _objects
 import pytest
 
 
@@ -70,7 +71,7 @@ class TestAstroidUtils:
             import_, basename
         )
         node = astroid.extract_node(source)
-        basenames = astroid_utils.resolve_qualname(node.bases[0], node.basenames[0])
+        basenames = _astroid_utils.resolve_qualname(node.bases[0], node.basenames[0])
         assert basenames == expected
 
     @pytest.mark.parametrize(
@@ -85,7 +86,7 @@ class TestAstroidUtils:
             import_, basename
         )
         node = astroid.extract_node(source)
-        basenames = astroid_utils.resolve_qualname(node.bases[0], node.basenames[0])
+        basenames = _astroid_utils.resolve_qualname(node.bases[0], node.basenames[0])
         assert basenames == expected
 
     @pytest.mark.parametrize(
@@ -99,7 +100,7 @@ class TestAstroidUtils:
     )
     def test_can_get_assign_values(self, source, expected):
         node = astroid.extract_node(source)
-        value = astroid_utils.get_assign_value(node)
+        value = _astroid_utils.get_assign_value(node)
         assert value == expected
 
     @pytest.mark.parametrize(
@@ -160,7 +161,7 @@ class TestAstroidUtils:
             )
         )
 
-        annotations = astroid_utils.get_args_info(node.args)
+        annotations = _astroid_utils.get_args_info(node.args)
         assert annotations == expected
 
     def test_parse_split_type_comments(self):
@@ -174,7 +175,7 @@ class TestAstroidUtils:
         """
         )
 
-        annotations = astroid_utils.get_args_info(node.args)
+        annotations = _astroid_utils.get_args_info(node.args)
 
         expected = [
             (None, "a", "int", None),
@@ -224,6 +225,6 @@ class TestAstroidUtils:
             )
         )
 
-        args_info = astroid_utils.get_args_info(node.args)
+        args_info = _astroid_utils.get_args_info(node.args)
-        formatted = objects._format_args(args_info)
+        formatted = _objects._format_args(args_info)
         assert formatted == expected