diff --git a/autoapi/mappers/python/astroid_utils.py b/autoapi/_astroid_utils.py similarity index 99% rename from autoapi/mappers/python/astroid_utils.py rename to autoapi/_astroid_utils.py index e647cfc..d15b6fb 100644 --- a/autoapi/mappers/python/astroid_utils.py +++ b/autoapi/_astroid_utils.py @@ -4,11 +4,7 @@ import re import astroid import astroid.nodes - import astroid.nodes.node_classes -import sphinx.util.logging - -_LOGGER = sphinx.util.logging.getLogger(__name__) def resolve_import_alias(name, import_names): diff --git a/autoapi/mappers/python/mapper.py b/autoapi/_mapper.py similarity index 73% rename from autoapi/mappers/python/mapper.py rename to autoapi/_mapper.py index 34e972a..b2454cb 100644 --- a/autoapi/mappers/python/mapper.py +++ b/autoapi/_mapper.py @@ -1,20 +1,23 @@ import collections import copy +import fnmatch import operator import os import re +from jinja2 import Environment, FileSystemLoader +import sphinx import sphinx.environment from sphinx.errors import ExtensionError import sphinx.util +import sphinx.util.logging from sphinx.util.console import colorize from sphinx.util.display import status_iterator import sphinx.util.docstrings -import sphinx.util.logging +from sphinx.util.osutil import ensuredir -from ..base import SphinxMapperBase -from .parser import Parser -from .objects import ( +from ._parser import Parser +from ._objects import ( PythonClass, PythonFunction, PythonModule, @@ -26,6 +29,7 @@ from .objects import ( PythonException, TopLevelPythonPythonMapper, ) +from .settings import OWN_PAGE_LEVELS, TEMPLATE_DIR LOGGER = sphinx.util.logging.getLogger(__name__) @@ -219,13 +223,11 @@ def _link_objs(value): return result[:-2] -class PythonSphinxMapper(SphinxMapperBase): - """AutoAPI domain handler for Python - - Parses directly from Python files. +class Mapper: + """Base class for mapping `PythonMapperBase` objects to Sphinx. Args: - app: Sphinx application passed in as part of the extension + app: Sphinx application instance """ _OBJ_MAP = { @@ -244,13 +246,142 @@ class PythonSphinxMapper(SphinxMapperBase): } def __init__(self, app, template_dir=None, dir_root=None, url_root=None): - super().__init__(app, template_dir, dir_root, url_root) + self.app = app + + template_paths = [TEMPLATE_DIR] + + if template_dir: + # Put at the front so it's loaded first + template_paths.insert(0, template_dir) + + self.jinja_env = Environment( + loader=FileSystemLoader(template_paths), + trim_blocks=True, + lstrip_blocks=True, + ) + + def _wrapped_prepare(value): + return value + + self.jinja_env.filters["prepare_docstring"] = _wrapped_prepare + if self.app.config.autoapi_prepare_jinja_env: + self.app.config.autoapi_prepare_jinja_env(self.jinja_env) + + own_page_level = self.app.config.autoapi_own_page_level + desired_page_level = OWN_PAGE_LEVELS.index(own_page_level) + self.own_page_types = set(OWN_PAGE_LEVELS[: desired_page_level + 1]) + + self.dir_root = dir_root + self.url_root = url_root + + # Mapping of {filepath -> raw data} + self.paths = collections.OrderedDict() + # Mapping of {object id -> Python Object} + self.objects_to_render = collections.OrderedDict() + # Mapping of {object id -> Python Object} + self.all_objects = collections.OrderedDict() + # Mapping of {namespace id -> Python Object} + self.namespaces = collections.OrderedDict() self.jinja_env.filters["link_objs"] = _link_objs self._use_implicit_namespace = ( self.app.config.autoapi_python_use_implicit_namespaces ) + @staticmethod + def find_files(patterns, dirs, ignore): + if not ignore: + ignore = [] + + pattern_regexes = [] + for pattern in patterns: + regex = re.compile(fnmatch.translate(pattern).replace(".*", "(.*)")) + pattern_regexes.append((pattern, regex)) + + for _dir in dirs: + for root, _, filenames in os.walk(_dir): + seen = set() + for pattern, pattern_re in pattern_regexes: + for filename in fnmatch.filter(filenames, pattern): + skip = False + + match = re.match(pattern_re, filename) + norm_name = match.groups() + if norm_name in seen: + continue + + # Skip ignored files + for ignore_pattern in ignore: + if fnmatch.fnmatch( + os.path.join(root, filename), ignore_pattern + ): + LOGGER.info( + colorize("bold", "[AutoAPI] ") + + colorize( + "darkgreen", f"Ignoring {root}/{filename}" + ) + ) + skip = True + + if skip: + continue + + # Make sure the path is full + if not os.path.isabs(filename): + filename = os.path.join(root, filename) + + yield filename + seen.add(norm_name) + + def add_object(self, obj): + """Add object to local and app environment storage + + Args: + obj: Instance of a AutoAPI object + """ + display = obj.display + if display and obj.type in self.own_page_types: + self.objects_to_render[obj.id] = obj + + self.all_objects[obj.id] = obj + child_stack = list(obj.children) + while child_stack: + child = child_stack.pop() + self.all_objects[child.id] = child + if display and child.type in self.own_page_types: + self.objects_to_render[child.id] = child + child_stack.extend(getattr(child, "children", ())) + + def output_rst(self, source_suffix): + for _, obj in status_iterator( + self.objects_to_render.items(), + colorize("bold", "[AutoAPI] ") + "Rendering Data... ", + length=len(self.objects_to_render), + verbosity=1, + stringify_func=(lambda x: x[0]), + ): + rst = obj.render(is_own_page=True) + if not rst: + continue + + output_dir = obj.output_dir(self.dir_root) + ensuredir(output_dir) + output_path = output_dir / obj.output_filename() + path = f"{output_path}{source_suffix}" + with open(path, "wb+") as detail_file: + detail_file.write(rst.encode("utf-8")) + + if self.app.config.autoapi_add_toctree_entry: + self._output_top_rst() + + def _output_top_rst(self): + # Render Top Index + top_level_index = os.path.join(self.dir_root, "index.rst") + pages = [obj for obj in self.objects_to_render.values() if obj.display] + with open(top_level_index, "wb") as top_level_file: + content = self.jinja_env.get_template("index.rst") + top_level_file.write(content.render(pages=pages).encode("utf-8")) + def _need_to_load(self, files): last_files = getattr(self.app.env, "autoapi_source_files", []) self.app.env.autoapi_source_files = files @@ -361,7 +492,14 @@ class PythonSphinxMapper(SphinxMapperBase): self._hide_yo_kids() self.app.env.autoapi_annotations = {} - super().map(options) + for _, data in status_iterator( + self.paths.items(), + colorize("bold", "[AutoAPI] ") + "Mapping Data... ", + length=len(self.paths), + stringify_func=(lambda x: x[0]), + ): + for obj in self.create_class(data, options=options): + self.add_object(obj) top_level_objects = { obj.id: obj @@ -383,7 +521,7 @@ class PythonSphinxMapper(SphinxMapperBase): self.app.env.autoapi_objects = self.objects_to_render self.app.env.autoapi_all_objects = self.all_objects - def create_class(self, data, options=None, **kwargs): + def create_class(self, data, options=None): """Create a class from the passed in data Args: @@ -402,13 +540,10 @@ class PythonSphinxMapper(SphinxMapperBase): jinja_env=self.jinja_env, app=self.app, url_root=self.url_root, - **kwargs, ) for child_data in data.get("children", []): - for child_obj in self.create_class( - child_data, options=options, **kwargs - ): + for child_obj in self.create_class(child_data, options=options): obj.children.append(child_obj) # Some objects require children to establish their docstring diff --git a/autoapi/mappers/python/objects.py b/autoapi/_objects.py similarity index 58% rename from autoapi/mappers/python/objects.py rename to autoapi/_objects.py index 42abf88..4e08b03 100644 --- a/autoapi/mappers/python/objects.py +++ b/autoapi/_objects.py @@ -1,11 +1,14 @@ +from __future__ import annotations + import functools import pathlib -from typing import List, Optional +from typing import List, Optional, Tuple +import sphinx +import sphinx.util import sphinx.util.logging -from ..base import PythonMapperBase - +from .settings import OWN_PAGE_LEVELS LOGGER = sphinx.util.logging.getLogger(__name__) @@ -27,105 +30,176 @@ def _format_args(args_info, include_annotations=True, ignore_self=None): return ", ".join(result) -class PythonPythonMapper(PythonMapperBase): - """A base class for all types of representations of Python objects. +class PythonObject: + """A class representing an entity from the parsed source code. - Attributes: - name (str): The name given to this object. - id (str): A unique identifier for this object. - children (list(PythonPythonMapper)): The members of this object. + This class turns the dictionaries output by the parser into an object. + + Args: + obj: JSON object representing this object + jinja_env: A template environment for rendering this object """ - language = "python" - is_callable = False member_order = 0 + """The ordering of objects when doing "groupwise" sorting.""" type: str - def __init__(self, obj, class_content="class", **kwargs) -> None: - super().__init__(obj, **kwargs) + def __init__( + self, obj, jinja_env, app, url_root, options=None, class_content="class" + ): + self.app = app + self.obj = obj + self.options = options + self.jinja_env = jinja_env + self.url_root = url_root + + self.name: str = obj["name"] + """The name of the object, as named in the parsed source code. - self.name = obj["name"] - self.qual_name = obj["qual_name"] - self.id = obj.get("full_name", self.name) + This name will have no periods in it. + """ + self.qual_name: str = obj["qual_name"] + """The qualified name for this object.""" + self.id: str = obj.get("full_name", self.name) + """A globally unique identifier for this object. + + This is the same as the fully qualified name of the object. + """ - # Optional - self.children: List[PythonPythonMapper] = [] - self._docstring = obj["doc"] - self._docstring_resolved = False - self.imported = "original_path" in obj - self.inherited = obj.get("inherited", False) - """Whether this was inherited from an ancestor of the parent class. + self.children: List[PythonObject] = [] + """The members of this object. - :type: bool + For example, the classes and functions defined in the parent module. """ + self._docstring: str = obj["doc"] + self.imported: bool = "original_path" in obj + """Whether this object was imported from another module.""" + self.inherited: bool = obj.get("inherited", False) + """Whether this was inherited from an ancestor of the parent class.""" self._hide = obj.get("hide", False) # For later self._class_content = class_content - self._display_cache: Optional[bool] = None + def __getstate__(self): + """Obtains serialisable data for pickling.""" + __dict__ = self.__dict__.copy() + __dict__.update(app=None, jinja_env=None) # clear unpickable attributes + return __dict__ + + def render(self, **kwargs): + LOGGER.log("VERBOSE", "Rendering %s", self.id) + + template = self.jinja_env.get_template(f"python/{self.type}.rst") + + ctx = {} + ctx.update(**self.get_context_data()) + ctx.update(**kwargs) + return template.render(**ctx) + + @property + def rendered(self): + """Shortcut to render an object in templates.""" + return self.render() + + def get_context_data(self): + own_page_level = self.app.config.autoapi_own_page_level + desired_page_level = OWN_PAGE_LEVELS.index(own_page_level) + own_page_types = set(OWN_PAGE_LEVELS[: desired_page_level + 1]) + + return { + "autoapi_options": self.app.config.autoapi_options, + "include_summaries": self.app.config.autoapi_include_summaries, + "obj": self, + "own_page_types": own_page_types, + "sphinx_version": sphinx.version_info, + } + + def __lt__(self, other): + """Object sorting comparison""" + if not isinstance(other, PythonObject): + return NotImplemented + + return self.id < other.id + + def __str__(self) -> str: + return f"<{self.__class__.__name__} {self.id}>" + + @property + def short_name(self) -> str: + """Shorten name property""" + return self.name.split(".")[-1] + + def output_dir(self, root): + """The directory to render this object.""" + module = self.id[: -(len("." + self.qual_name))] + parts = [root] + module.split(".") + return pathlib.PurePosixPath(*parts) + + def output_filename(self) -> str: + """The name of the file to render into, without a file suffix.""" + filename = self.qual_name + if filename == "index": + filename = ".index" + + return filename + + @property + def include_path(self) -> str: + """Return 'absolute' path without regarding OS path separator + + This is used in ``toctree`` directives, as Sphinx always expects Unix + path separators + """ + return str(self.output_dir(self.url_root) / self.output_filename()) + @property - def docstring(self): + def docstring(self) -> str: """The docstring for this object. If a docstring did not exist on the object, this will be the empty string. - For classes this will also depend on the + For classes, this will also depend on the :confval:`autoapi_python_class_content` option. - - :type: str """ return self._docstring @docstring.setter - def docstring(self, value): + def docstring(self, value: str) -> None: self._docstring = value self._docstring_resolved = True @property - def is_top_level_object(self): + def is_top_level_object(self) -> bool: """Whether this object is at the very top level (True) or not (False). This will be False for subpackages and submodules. - - :type: bool """ return "." not in self.id @property - def is_undoc_member(self): - """Whether this object has a docstring (False) or not (True). - - :type: bool - """ + def is_undoc_member(self) -> bool: + """Whether this object has a docstring (False) or not (True).""" return not bool(self.docstring) @property - def is_private_member(self): - """Whether this object is private (True) or not (False). - - :type: bool - """ + def is_private_member(self) -> bool: + """Whether this object is private (True) or not (False).""" return self.short_name.startswith("_") and not self.short_name.endswith("__") @property - def is_special_member(self): - """Whether this object is a special member (True) or not (False). - - :type: bool - """ + def is_special_member(self) -> bool: + """Whether this object is a special member (True) or not (False).""" return self.short_name.startswith("__") and self.short_name.endswith("__") @property - def display(self): + def display(self) -> bool: """Whether this object should be displayed in documentation. This attribute depends on the configuration options given in :confval:`autoapi_options` and the result of :event:`autoapi-skip-member`. - - :type: bool """ if self._display_cache is None: self._display_cache = not self._ask_ignore(self._should_skip()) @@ -133,13 +207,11 @@ class PythonPythonMapper(PythonMapperBase): return self._display_cache @property - def summary(self): + def summary(self) -> str: """The summary line of the docstring. The summary line is the first non-empty line, as-per :pep:`257`. This will be the empty string if the object does not have a docstring. - - :type: str """ for line in self.docstring.splitlines(): line = line.strip() @@ -148,7 +220,7 @@ class PythonPythonMapper(PythonMapperBase): return "" - def _should_skip(self): # type: () -> bool + def _should_skip(self) -> bool: skip_undoc_member = self.is_undoc_member and "undoc-members" not in self.options skip_private_member = ( self.is_private_member and "private-members" not in self.options @@ -170,61 +242,53 @@ class PythonPythonMapper(PythonMapperBase): or skip_inherited_member ) - def _ask_ignore(self, skip): # type: (bool) -> bool + def _ask_ignore(self, skip: bool) -> bool: ask_result = self.app.emit_firstresult( "autoapi-skip-member", self.type, self.id, self, skip, self.options ) return ask_result if ask_result is not None else skip - def _children_of_type(self, type_): + def _children_of_type(self, type_: str) -> List[PythonObject]: return list(child for child in self.children if child.type == type_) -class PythonFunction(PythonPythonMapper): +class PythonFunction(PythonObject): """The representation of a function.""" type = "function" - is_callable = True member_order = 30 - def __init__(self, obj, **kwargs): - super().__init__(obj, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) autodoc_typehints = getattr(self.app.config, "autodoc_typehints", "signature") show_annotations = autodoc_typehints != "none" and not ( - autodoc_typehints == "description" and not obj["overloads"] + autodoc_typehints == "description" and not self.obj["overloads"] ) - self.args = _format_args(obj["args"], show_annotations) - """The arguments to this object, formatted as a string. + self.args: str = _format_args(self.obj["args"], show_annotations) + """The arguments to this object, formatted as a string.""" - :type: str - """ - - self.return_annotation = obj["return_annotation"] if show_annotations else None + self.return_annotation: Optional[str] = ( + self.obj["return_annotation"] if show_annotations else None + ) """The type annotation for the return type of this function. This will be ``None`` if an annotation or annotation comment was not given. - - :type: str or None """ - self.properties = obj["properties"] + self.properties: List[str] = self.obj["properties"] """The properties that describe what type of function this is. - Can be only be: async - - :type: list(str) + Can be only be: async. """ - self.overloads = [ + self.overloads: List[Tuple[str, str]] = [ (_format_args(args), return_annotation) - for args, return_annotation in obj["overloads"] + for args, return_annotation in self.obj["overloads"] ] """The overloaded signatures of this function. Each tuple is a tuple of ``(args, return_annotation)`` - - :type: list(tuple(str, str)) """ @@ -232,73 +296,61 @@ class PythonMethod(PythonFunction): """The representation of a method.""" type = "method" - is_callable = True member_order = 50 - def __init__(self, obj, **kwargs): - super().__init__(obj, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) - self.properties = obj["properties"] + self.properties: List[str] = self.obj["properties"] """The properties that describe what type of method this is. - Can be any of: abstractmethod, async, classmethod, property, staticmethod - - :type: list(str) + Can be any of: abstractmethod, async, classmethod, property, staticmethod. """ - def _should_skip(self): # type: () -> bool + def _should_skip(self) -> bool: return super()._should_skip() or self.name in ( "__new__", "__init__", ) -class PythonProperty(PythonPythonMapper): +class PythonProperty(PythonObject): """The representation of a property on a class.""" type = "property" member_order = 60 - def __init__(self, obj, **kwargs): - super().__init__(obj, **kwargs) - - self.annotation = obj["return_annotation"] - """The type annotation of this property. + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) - :type: str or None - """ - self.properties = obj["properties"] + self.annotation: Optional[str] = self.obj["return_annotation"] + """The type annotation of this property.""" + self.properties: List[str] = self.obj["properties"] """The properties that describe what type of property this is. - Can be any of: abstractmethod, classmethod - - :type: list(str) + Can be any of: abstractmethod, classmethod. """ -class PythonData(PythonPythonMapper): +class PythonData(PythonObject): """Global, module level data.""" type = "data" member_order = 40 - def __init__(self, obj, **kwargs): - super().__init__(obj, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) - self.value = obj.get("value") + self.value: Optional[str] = self.obj.get("value") """The value of this attribute. This will be ``None`` if the value is not constant. - - :type: str or None """ - self.annotation = obj.get("annotation") + self.annotation: Optional[str] = self.obj.get("annotation") """The type annotation of this attribute. This will be ``None`` if an annotation or annotation comment was not given. - - :type: str or None """ @@ -309,17 +361,15 @@ class PythonAttribute(PythonData): member_order = 60 -class TopLevelPythonPythonMapper(PythonPythonMapper): +class TopLevelPythonPythonMapper(PythonObject): """A common base class for modules and packages.""" - _RENDER_LOG_LEVEL = "VERBOSE" - - def __init__(self, obj, **kwargs): - super().__init__(obj, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.subpackages = [] self.submodules = [] - self.all = obj["all"] + self.all = self.obj["all"] """The contents of ``__all__`` if assigned to. Only constants are included. @@ -366,27 +416,23 @@ class PythonPackage(TopLevelPythonPythonMapper): type = "package" -class PythonClass(PythonPythonMapper): +class PythonClass(PythonObject): """The representation of a class.""" type = "class" member_order = 20 - def __init__(self, obj, **kwargs): - super().__init__(obj, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) - self.bases = obj["bases"] - """The fully qualified names of all base classes. + self.bases: List[str] = self.obj["bases"] + """The fully qualified names of all base classes.""" - :type: list(str) - """ + self._docstring_resolved: bool = False @property - def args(self): - """The arguments to this object, formatted as a string. - - :type: str - """ + def args(self) -> str: + """The arguments to this object, formatted as a string.""" args = "" if self.constructor: @@ -402,7 +448,7 @@ class PythonClass(PythonPythonMapper): return args @property - def overloads(self): + def overloads(self) -> List[Tuple[str, str]]: overloads = [] if self.constructor: @@ -422,8 +468,8 @@ class PythonClass(PythonPythonMapper): return overloads @property - def docstring(self): - docstring = super().docstring + def docstring(self) -> str: + docstring = self._docstring if not self._docstring_resolved and self._class_content in ("both", "init"): constructor_docstring = self.constructor_docstring @@ -437,8 +483,9 @@ class PythonClass(PythonPythonMapper): return docstring @docstring.setter - def docstring(self, value): - super(PythonClass, self.__class__).docstring.fset(self, value) + def docstring(self, value: str) -> None: + self._docstring = value + self._docstring_resolved = True @property def methods(self): @@ -466,7 +513,7 @@ class PythonClass(PythonPythonMapper): return None @property - def constructor_docstring(self): + def constructor_docstring(self) -> str: docstring = "" constructor = self.constructor diff --git a/autoapi/mappers/python/parser.py b/autoapi/_parser.py similarity index 89% rename from autoapi/mappers/python/parser.py rename to autoapi/_parser.py index 22bf8d9..3c347e0 100644 --- a/autoapi/mappers/python/parser.py +++ b/autoapi/_parser.py @@ -6,7 +6,7 @@ import astroid import astroid.builder import sphinx.util.docstrings -from . import astroid_utils +from . import _astroid_utils def _prepare_docstring(doc): @@ -80,17 +80,17 @@ class Parser: type_ = "data" if isinstance( node.scope(), astroid.nodes.ClassDef - ) or astroid_utils.is_constructor(node.scope()): + ) or _astroid_utils.is_constructor(node.scope()): type_ = "attribute" - assign_value = astroid_utils.get_assign_value(node) + assign_value = _astroid_utils.get_assign_value(node) if not assign_value: return [] target = assign_value[0] value = assign_value[1] - annotation = astroid_utils.get_assign_annotation(node) + annotation = _astroid_utils.get_assign_annotation(node) data = { "type": type_, @@ -108,10 +108,10 @@ class Parser: def parse_classdef(self, node, data=None): type_ = "class" - if astroid_utils.is_exception(node): + if _astroid_utils.is_exception(node): type_ = "exception" - basenames = list(astroid_utils.get_full_basenames(node)) + basenames = list(_astroid_utils.get_full_basenames(node)) data = { "type": type_, @@ -119,7 +119,7 @@ class Parser: "qual_name": self._get_qual_name(node.name), "full_name": self._get_full_name(node.name), "bases": basenames, - "doc": _prepare_docstring(astroid_utils.get_class_docstring(node)), + "doc": _prepare_docstring(_astroid_utils.get_class_docstring(node)), "from_line_no": node.fromlineno, "to_line_no": node.tolineno, "children": [], @@ -140,7 +140,7 @@ class Parser: for child in base.get_children(): name = getattr(child, "name", None) if isinstance(child, (astroid.Assign, astroid.AnnAssign)): - assign_value = astroid_utils.get_assign_value(child) + assign_value = _astroid_utils.get_assign_value(child) if not assign_value: continue name = assign_value[0] @@ -164,7 +164,7 @@ class Parser: return self.parse_functiondef(node) def parse_functiondef(self, node): - if astroid_utils.is_decorated_with_property_setter(node): + if _astroid_utils.is_decorated_with_property_setter(node): return [] type_ = "method" @@ -175,7 +175,7 @@ class Parser: if isinstance(node, astroid.AsyncFunctionDef): properties.append("async") - elif astroid_utils.is_decorated_with_property(node): + elif _astroid_utils.is_decorated_with_property(node): type_ = "property" if node.type == "classmethod": properties.append(node.type) @@ -195,13 +195,13 @@ class Parser: "name": node.name, "qual_name": self._get_qual_name(node.name), "full_name": self._get_full_name(node.name), - "args": astroid_utils.get_args_info(node.args), - "doc": _prepare_docstring(astroid_utils.get_func_docstring(node)), + "args": _astroid_utils.get_args_info(node.args), + "doc": _prepare_docstring(_astroid_utils.get_func_docstring(node)), "from_line_no": node.fromlineno, "to_line_no": node.tolineno, - "return_annotation": astroid_utils.get_return_annotation(node), + "return_annotation": _astroid_utils.get_return_annotation(node), "properties": properties, - "is_overload": astroid_utils.is_decorated_with_overload(node), + "is_overload": _astroid_utils.is_decorated_with_overload(node), "overloads": [], } @@ -220,7 +220,7 @@ class Parser: for import_name, alias in node.names: is_wildcard = (alias or import_name) == "*" - original_path = astroid_utils.get_full_import_name( + original_path = _astroid_utils.get_full_import_name( node, alias or import_name ) name = original_path if is_wildcard else (alias or import_name) @@ -259,13 +259,13 @@ class Parser: "children": [], "file_path": path, "encoding": node.file_encoding, - "all": astroid_utils.get_module_all(node), + "all": _astroid_utils.get_module_all(node), } overloads = {} top_name = node.name.split(".", 1)[0] for child in node.get_children(): - if astroid_utils.is_local_import_from(child, top_name): + if _astroid_utils.is_local_import_from(child, top_name): child_data = self._parse_local_import_from(child) else: child_data = self.parse(child) diff --git a/autoapi/directives.py b/autoapi/directives.py index 8eebf30..7f26c56 100644 --- a/autoapi/directives.py +++ b/autoapi/directives.py @@ -6,7 +6,7 @@ from docutils import nodes from sphinx.ext.autosummary import Autosummary, mangle_signature from sphinx.util.nodes import nested_parse_with_titles -from .mappers.python.objects import PythonFunction +from ._objects import PythonFunction class AutoapiSummary(Autosummary): diff --git a/autoapi/documenters.py b/autoapi/documenters.py index adf27ec..44c0687 100644 --- a/autoapi/documenters.py +++ b/autoapi/documenters.py @@ -2,7 +2,7 @@ import re from sphinx.ext import autodoc -from .mappers.python import ( +from ._objects import ( PythonFunction, PythonClass, PythonMethod, diff --git a/autoapi/extension.py b/autoapi/extension.py index 00a1ffd..6c11413 100644 --- a/autoapi/extension.py +++ b/autoapi/extension.py @@ -19,7 +19,7 @@ from docutils.parsers.rst import directives from . import documenters from .directives import AutoapiSummary, NestedParse from .inheritance_diagrams import AutoapiInheritanceDiagram -from .mappers import PythonSphinxMapper +from ._mapper import Mapper from .settings import API_ROOT LOGGER = sphinx.util.logging.getLogger(__name__) @@ -111,7 +111,7 @@ def run_autoapi(app): "relative to where sphinx-build is run\n", RemovedInAutoAPI3Warning, ) - sphinx_mapper_obj = PythonSphinxMapper( + sphinx_mapper_obj = Mapper( app, template_dir=template_dir, dir_root=normalized_root, url_root=url_root ) diff --git a/autoapi/mappers/__init__.py b/autoapi/mappers/__init__.py deleted file mode 100644 index e19c509..0000000 --- a/autoapi/mappers/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .python import PythonSphinxMapper - -__all__ = ("PythonSphinxMapper",) diff --git a/autoapi/mappers/base.py b/autoapi/mappers/base.py deleted file mode 100644 index e55413e..0000000 --- a/autoapi/mappers/base.py +++ /dev/null @@ -1,342 +0,0 @@ -from collections import OrderedDict, namedtuple -import fnmatch -import os -import pathlib -import re - -from jinja2 import Environment, FileSystemLoader, TemplateNotFound -import sphinx -import sphinx.util -from sphinx.util.console import colorize -from sphinx.util.display import status_iterator -from sphinx.util.osutil import ensuredir -import sphinx.util.logging - -from ..settings import TEMPLATE_DIR - -LOGGER = sphinx.util.logging.getLogger(__name__) -_OWN_PAGE_LEVELS = [ - "package", - "module", - "exception", - "class", - "function", - "method", - "property", - "data", - "attribute", -] - -Path = namedtuple("Path", ["absolute", "relative"]) - - -class PythonMapperBase: - """Base object for JSON -> Python object mapping. - - Subclasses of this object will handle their language specific JSON input, - and map that onto this standard Python object. - Subclasses may also include language-specific attributes on this object. - - Args: - obj: JSON object representing this object - jinja_env: A template environment for rendering this object - - Attributes: - id (str): A globally unique identifier for this object. - Generally a fully qualified name, including namespace. - name (str): A short "display friendly" name for this object. - docstring (str): The documentation for this object - imports (list): Imports in this object - children (list): Children of this object - parameters (list): Parameters to this object - methods (list): Methods on this object - """ - - language = "base" - type = "base" - _RENDER_LOG_LEVEL = "VERBOSE" - - def __init__(self, obj, jinja_env, app, url_root, options=None): - self.app = app - self.obj = obj - self.options = options - self.jinja_env = jinja_env - self.url_root = url_root - - self.name = None - self.qual_name = None - self.id = None - - def __getstate__(self): - """Obtains serialisable data for pickling.""" - __dict__ = self.__dict__.copy() - __dict__.update(app=None, jinja_env=None) # clear unpickable attributes - return __dict__ - - def render(self, **kwargs): - LOGGER.log(self._RENDER_LOG_LEVEL, "Rendering %s", self.id) - - ctx = {} - try: - template = self.jinja_env.get_template(f"{self.language}/{self.type}.rst") - except TemplateNotFound: - template = self.jinja_env.get_template(f"base/{self.type}.rst") - - ctx.update(**self.get_context_data()) - ctx.update(**kwargs) - return template.render(**ctx) - - @property - def rendered(self): - """Shortcut to render an object in templates.""" - return self.render() - - def get_context_data(self): - own_page_level = self.app.config.autoapi_own_page_level - desired_page_level = _OWN_PAGE_LEVELS.index(own_page_level) - own_page_types = set(_OWN_PAGE_LEVELS[: desired_page_level + 1]) - - return { - "autoapi_options": self.app.config.autoapi_options, - "include_summaries": self.app.config.autoapi_include_summaries, - "obj": self, - "own_page_types": own_page_types, - "sphinx_version": sphinx.version_info, - } - - def __lt__(self, other): - """Object sorting comparison""" - if isinstance(other, PythonMapperBase): - return self.id < other.id - return super().__lt__(other) - - def __str__(self): - return f"<{self.__class__.__name__} {self.id}>" - - @property - def short_name(self): - """Shorten name property""" - return self.name.split(".")[-1] - - def output_dir(self, root): - """The directory to render this object.""" - module = self.id[: -(len("." + self.qual_name))] - parts = [root] + module.split(".") - return pathlib.PurePosixPath(*parts) - - def output_filename(self): - """The name of the file to render into, without a file suffix.""" - filename = self.qual_name - if filename == "index": - filename = ".index" - - return filename - - @property - def include_path(self): - """Return 'absolute' path without regarding OS path separator - - This is used in ``toctree`` directives, as Sphinx always expects Unix - path separators - """ - return str(self.output_dir(self.url_root) / self.output_filename()) - - @property - def display(self): - """Whether to display this object or not. - - :type: bool - """ - return True - - @property - def ref_type(self): - return self.type - - @property - def ref_directive(self): - return self.type - - -class SphinxMapperBase: - """Base class for mapping `PythonMapperBase` objects to Sphinx. - - Args: - app: Sphinx application instance - """ - - def __init__(self, app, template_dir=None, dir_root=None, url_root=None): - self.app = app - - template_paths = [TEMPLATE_DIR] - - if template_dir: - # Put at the front so it's loaded first - template_paths.insert(0, template_dir) - - self.jinja_env = Environment( - loader=FileSystemLoader(template_paths), - trim_blocks=True, - lstrip_blocks=True, - ) - - def _wrapped_prepare(value): - return value - - self.jinja_env.filters["prepare_docstring"] = _wrapped_prepare - if self.app.config.autoapi_prepare_jinja_env: - self.app.config.autoapi_prepare_jinja_env(self.jinja_env) - - own_page_level = self.app.config.autoapi_own_page_level - desired_page_level = _OWN_PAGE_LEVELS.index(own_page_level) - self.own_page_types = set(_OWN_PAGE_LEVELS[: desired_page_level + 1]) - - self.dir_root = dir_root - self.url_root = url_root - - # Mapping of {filepath -> raw data} - self.paths = OrderedDict() - # Mapping of {object id -> Python Object} - self.objects_to_render = OrderedDict() - # Mapping of {object id -> Python Object} - self.all_objects = OrderedDict() - # Mapping of {namespace id -> Python Object} - self.namespaces = OrderedDict() - - def load(self, patterns, dirs, ignore=None): - """Load objects from the filesystem into the ``paths`` dictionary.""" - paths = list(self.find_files(patterns=patterns, dirs=dirs, ignore=ignore)) - for path in status_iterator( - paths, - colorize("bold", "[AutoAPI] Reading files... "), - "darkgreen", - len(paths), - ): - data = self.read_file(path=path) - if data: - self.paths[path] = data - - return True - - @staticmethod - def find_files(patterns, dirs, ignore): - if not ignore: - ignore = [] - - pattern_regexes = [] - for pattern in patterns: - regex = re.compile(fnmatch.translate(pattern).replace(".*", "(.*)")) - pattern_regexes.append((pattern, regex)) - - for _dir in dirs: - for root, _, filenames in os.walk(_dir): - seen = set() - for pattern, pattern_re in pattern_regexes: - for filename in fnmatch.filter(filenames, pattern): - skip = False - - match = re.match(pattern_re, filename) - norm_name = match.groups() - if norm_name in seen: - continue - - # Skip ignored files - for ignore_pattern in ignore: - if fnmatch.fnmatch( - os.path.join(root, filename), ignore_pattern - ): - LOGGER.info( - colorize("bold", "[AutoAPI] ") - + colorize( - "darkgreen", f"Ignoring {root}/{filename}" - ) - ) - skip = True - - if skip: - continue - - # Make sure the path is full - if not os.path.isabs(filename): - filename = os.path.join(root, filename) - - yield filename - seen.add(norm_name) - - def read_file(self, path, **kwargs): - """Read file input into memory - - Args: - path: Path of file to read - """ - # TODO support JSON here - # TODO sphinx way of reporting errors in logs? - raise NotImplementedError - - def add_object(self, obj): - """Add object to local and app environment storage - - Args: - obj: Instance of a AutoAPI object - """ - display = obj.display - if display and obj.type in self.own_page_types: - self.objects_to_render[obj.id] = obj - - self.all_objects[obj.id] = obj - child_stack = list(obj.children) - while child_stack: - child = child_stack.pop() - self.all_objects[child.id] = child - if display and child.type in self.own_page_types: - self.objects_to_render[child.id] = child - child_stack.extend(getattr(child, "children", ())) - - def map(self, options=None): - """Trigger find of serialized sources and build objects""" - for _, data in status_iterator( - self.paths.items(), - colorize("bold", "[AutoAPI] ") + "Mapping Data... ", - length=len(self.paths), - stringify_func=(lambda x: x[0]), - ): - for obj in self.create_class(data, options=options): - self.add_object(obj) - - def create_class(self, data, options=None, **kwargs): - """Create class object. - - Args: - data: Instance of a AutoAPI object - """ - raise NotImplementedError - - def output_rst(self, source_suffix): - for _, obj in status_iterator( - self.objects_to_render.items(), - colorize("bold", "[AutoAPI] ") + "Rendering Data... ", - length=len(self.objects_to_render), - verbosity=1, - stringify_func=(lambda x: x[0]), - ): - rst = obj.render(is_own_page=True) - if not rst: - continue - - output_dir = obj.output_dir(self.dir_root) - ensuredir(output_dir) - output_path = output_dir / obj.output_filename() - path = f"{output_path}{source_suffix}" - with open(path, "wb+") as detail_file: - detail_file.write(rst.encode("utf-8")) - - if self.app.config.autoapi_add_toctree_entry: - self._output_top_rst() - - def _output_top_rst(self): - # Render Top Index - top_level_index = os.path.join(self.dir_root, "index.rst") - pages = [obj for obj in self.objects_to_render.values() if obj.display] - with open(top_level_index, "wb") as top_level_file: - content = self.jinja_env.get_template("index.rst") - top_level_file.write(content.render(pages=pages).encode("utf-8")) diff --git a/autoapi/mappers/python/__init__.py b/autoapi/mappers/python/__init__.py deleted file mode 100644 index c78a506..0000000 --- a/autoapi/mappers/python/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from .mapper import PythonSphinxMapper -from .objects import ( - PythonAttribute, - PythonClass, - PythonData, - PythonException, - PythonFunction, - PythonMethod, - PythonModule, - PythonPackage, - PythonProperty, -) - -__all__ = ( - "PythonAttribute", - "PythonClass", - "PythonData", - "PythonException", - "PythonFunction", - "PythonMethod", - "PythonModule", - "PythonPackage", - "PythonProperty", - "PythonSphinxMapper", -) diff --git a/autoapi/settings.py b/autoapi/settings.py index 58704b4..722fb33 100644 --- a/autoapi/settings.py +++ b/autoapi/settings.py @@ -10,3 +10,14 @@ SITE_ROOT = os.path.dirname(os.path.realpath(__file__)) TEMPLATE_DIR = os.path.join(SITE_ROOT, "templates") API_ROOT = "autoapi" +OWN_PAGE_LEVELS = [ + "package", + "module", + "exception", + "class", + "function", + "method", + "property", + "data", + "attribute", +] diff --git a/autoapi/templates/base/base.rst b/autoapi/templates/base/base.rst deleted file mode 100644 index 45c8dd7..0000000 --- a/autoapi/templates/base/base.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. {{ obj.type }}:: {{ obj.name }} - - {% if summary %} - - {{ obj.summary }} - - {% endif %} diff --git a/docs/changes/+cf0a1267.misc b/docs/changes/+cf0a1267.misc new file mode 100644 index 0000000..a390c4c --- /dev/null +++ b/docs/changes/+cf0a1267.misc @@ -0,0 +1 @@ +Refactor mapper classes into their bases \ No newline at end of file diff --git a/docs/reference/directives.rst b/docs/reference/directives.rst index 69ea793..3aedb20 100644 --- a/docs/reference/directives.rst +++ b/docs/reference/directives.rst @@ -55,7 +55,7 @@ Inheritance Diagrams For example: - .. autoapi-inheritance-diagram:: autoapi.mappers.python.objects.PythonModule autoapi.mappers.python.objects.PythonPackage + .. autoapi-inheritance-diagram:: autoapi._objects.PythonModule autoapi._objects.PythonPackage :parts: 1 :mod:`sphinx.ext.inheritance_diagram` makes use of the diff --git a/tests/python/test_parser.py b/tests/python/test_parser.py index 155d658..a7aa523 100644 --- a/tests/python/test_parser.py +++ b/tests/python/test_parser.py @@ -1,15 +1,8 @@ -# coding=utf8 - """Test Python parser""" -from io import StringIO -import sys -from textwrap import dedent - import astroid -import pytest -from autoapi.mappers.python.parser import Parser +from autoapi._parser import Parser class TestPythonParser: diff --git a/tests/python/test_pyintegration.py b/tests/python/test_pyintegration.py index e106fec..794ce48 100644 --- a/tests/python/test_pyintegration.py +++ b/tests/python/test_pyintegration.py @@ -5,7 +5,7 @@ import sys from unittest.mock import Mock, call import autoapi.settings -from autoapi.mappers.python import ( +from autoapi._objects import ( PythonClass, PythonData, PythonFunction, diff --git a/tests/test_astroid_utils.py b/tests/test_astroid_utils.py index e211638..7db157b 100644 --- a/tests/test_astroid_utils.py +++ b/tests/test_astroid_utils.py @@ -1,7 +1,8 @@ import sys import astroid -from autoapi.mappers.python import astroid_utils, objects +from autoapi import _astroid_utils +from autoapi import _objects import pytest @@ -70,7 +71,7 @@ class TestAstroidUtils: import_, basename ) node = astroid.extract_node(source) - basenames = astroid_utils.resolve_qualname(node.bases[0], node.basenames[0]) + basenames = _astroid_utils.resolve_qualname(node.bases[0], node.basenames[0]) assert basenames == expected @pytest.mark.parametrize( @@ -85,7 +86,7 @@ class TestAstroidUtils: import_, basename ) node = astroid.extract_node(source) - basenames = astroid_utils.resolve_qualname(node.bases[0], node.basenames[0]) + basenames = _astroid_utils.resolve_qualname(node.bases[0], node.basenames[0]) assert basenames == expected @pytest.mark.parametrize( @@ -99,7 +100,7 @@ class TestAstroidUtils: ) def test_can_get_assign_values(self, source, expected): node = astroid.extract_node(source) - value = astroid_utils.get_assign_value(node) + value = _astroid_utils.get_assign_value(node) assert value == expected @pytest.mark.parametrize( @@ -160,7 +161,7 @@ class TestAstroidUtils: ) ) - annotations = astroid_utils.get_args_info(node.args) + annotations = _astroid_utils.get_args_info(node.args) assert annotations == expected def test_parse_split_type_comments(self): @@ -174,7 +175,7 @@ class TestAstroidUtils: """ ) - annotations = astroid_utils.get_args_info(node.args) + annotations = _astroid_utils.get_args_info(node.args) expected = [ (None, "a", "int", None), @@ -224,6 +225,6 @@ class TestAstroidUtils: ) ) - args_info = astroid_utils.get_args_info(node.args) - formatted = objects._format_args(args_info) + args_info = _astroid_utils.get_args_info(node.args) + formatted = _objects._format_args(args_info) assert formatted == expected