Implmented basic incremental building

Closes #191

CHANGELOG.rst

@@ -17,6 +17,10 @@ Features
 
 * Added support for using type hints as parameter types and return types
   via the ``sphinx.ext.autodoc.typehints`` extension.
+* `#191 <https://github.com/readthedocs/sphinx-autoapi/issues/191>`:
+  Basic incremental build support is enabled ``autoapi_keep_files`` is enabled.
+	Providing none of the source files have changed,
+	AutoAPI will skip parsing the source code and regenerating the API documentation.
 
 Bug Fixes
 ^^^^^^^^^

autoapi/directives.py

@@ -15,10 +15,10 @@ class AutoapiSummary(Autosummary):  # pylint: disable=too-few-public-methods
     def get_items(self, names):
         items = []
         env = self.state.document.settings.env
-        mapper = env.autoapi_mapper
+        all_objects = env.autoapi_all_objects
 
         for name in names:
-            obj = mapper.all_objects[name]
+            obj = all_objects[name]
             if isinstance(obj, PythonFunction):
                 if obj.overloads:
                     sig = "(\u2026)"

autoapi/documenters.py

@@ -38,7 +38,7 @@ class AutoapiDocumenter(autodoc.Documenter):
         max_splits = self.fullname.count(".")
         for num_splits in range(max_splits, -1, -1):
             path_stack = list(reversed(self.fullname.rsplit(".", num_splits)))
-            objects = self.env.autoapi_mapper.objects
+            objects = self.env.autoapi_objects
             parent = None
             current = objects.get(path_stack.pop())
             while current and path_stack:

autoapi/extension.py

@@ -55,6 +55,20 @@ if "PYTHONWARNINGS" not in os.environ:
     warnings.filterwarnings("default", category=RemovedInAutoAPI2Warning)
 
 
+def _normalise_autoapi_dirs(autoapi_dirs, confdir):
+    normalised_dirs = []
+
+    if isinstance(autoapi_dirs, str):
+        autoapi_dirs = [autoapi_dirs]
+    for path in autoapi_dirs:
+        if os.path.isabs(path):
+            normalised_dirs.append(path)
+        else:
+            normalised_dirs.append(os.path.normpath(os.path.join(confdir, path)))
+
+    return normalised_dirs
+
+
 def run_autoapi(app):  # pylint: disable=too-many-branches
     """
     Load AutoAPI data from the filesystem.
@@ -82,17 +96,8 @@ def run_autoapi(app):  # pylint: disable=too-many-branches
             app.config.autoapi_options.append("show-module-summary")
 
     # Make sure the paths are full
-    normalized_dirs = []
-    autoapi_dirs = app.config.autoapi_dirs
-    if isinstance(autoapi_dirs, str):
-        autoapi_dirs = [autoapi_dirs]
-    for path in autoapi_dirs:
-        if os.path.isabs(path):
-            normalized_dirs.append(path)
-        else:
-            normalized_dirs.append(os.path.normpath(os.path.join(app.confdir, path)))
-
-    for _dir in normalized_dirs:
+    normalised_dirs = _normalise_autoapi_dirs(app.config.autoapi_dirs, app.confdir)
+    for _dir in normalised_dirs:
         if not os.path.exists(_dir):
             raise ExtensionError(
                 "AutoAPI Directory `{dir}` not found. "
@@ -137,7 +142,6 @@ def run_autoapi(app):  # pylint: disable=too-many-branches
                 RemovedInAutoAPI2Warning,
             )
     sphinx_mapper_obj = sphinx_mapper(app, template_dir=template_dir, url_root=url_root)
-    app.env.autoapi_mapper = sphinx_mapper_obj
 
     if app.config.autoapi_file_patterns:
         file_patterns = app.config.autoapi_file_patterns
@@ -157,15 +161,17 @@ def run_autoapi(app):  # pylint: disable=too-many-branches
         # Fallback to first suffix listed
         out_suffix = app.config.source_suffix[0]
 
-    # Actual meat of the run.
-    sphinx_mapper_obj.load(
-        patterns=file_patterns, dirs=normalized_dirs, ignore=ignore_patterns
-    )
+    if sphinx_mapper_obj.load(
+        patterns=file_patterns, dirs=normalised_dirs, ignore=ignore_patterns
+    ):
+        sphinx_mapper_obj.map(options=app.config.autoapi_options)
 
-    sphinx_mapper_obj.map(options=app.config.autoapi_options)
+        if app.config.autoapi_generate_api_docs:
+            sphinx_mapper_obj.output_rst(root=normalized_root, source_suffix=out_suffix)
 
-    if app.config.autoapi_generate_api_docs:
-        sphinx_mapper_obj.output_rst(root=normalized_root, source_suffix=out_suffix)
+        if app.config.autoapi_type == "python":
+            app.env.autoapi_objects = sphinx_mapper_obj.objects
+            app.env.autoapi_all_objects = sphinx_mapper_obj.all_objects
 
 
 def build_finished(app, exception):
@@ -220,21 +226,16 @@ def doctree_read(app, doctree):
             LOGGER.info(message_prefix + message)
 
 
-def clear_env(_, env):
-    """Clears the environment of the unpicklable objects that we left behind."""
-    env.autoapi_mapper = None
-
-
 def viewcode_find(app, modname):
-    mapper = app.env.autoapi_mapper
-    if modname not in mapper.objects:
+    objects = app.env.autoapi_objects
+    if modname not in objects:
         return None
 
     if modname in _VIEWCODE_CACHE:
         return _VIEWCODE_CACHE[modname]
 
     locations = {}
-    module = mapper.objects[modname]
+    module = objects[modname]
     for child in module.children:
         stack = [("", child)]
         while stack:
@@ -268,11 +269,11 @@ def viewcode_find(app, modname):
 
 def viewcode_follow_imported(app, modname, attribute):
     fullname = "{}.{}".format(modname, attribute)
-    mapper = app.env.autoapi_mapper
-    if fullname not in mapper.all_objects:
+    all_objects = app.env.autoapi_all_objects
+    if fullname not in all_objects:
         return None
 
-    orig_path = mapper.all_objects[fullname].obj.get("original_path", "")
+    orig_path = all_objects[fullname].obj.get("original_path", "")
     if orig_path.endswith(attribute):
         return orig_path[: -len(attribute) - 1]
 
@@ -283,7 +284,6 @@ def setup(app):
     app.connect("builder-inited", run_autoapi)
     app.connect("doctree-read", doctree_read)
     app.connect("build-finished", build_finished)
-    app.connect("env-updated", clear_env)
     if "viewcode-find-source" in app.events.events:
         app.connect("viewcode-find-source", viewcode_find)
     if "viewcode-follow-imported" in app.events.events:

autoapi/mappers/base.py

@@ -64,6 +64,12 @@ class PythonMapperBase(object):
         self.name = None
         self.id = None
 
+    def __getstate__(self):
+        """Obtains serialisable data for pickling."""
+        __dict__ = self.__dict__.copy()
+        __dict__.update(app=None, jinja_env=None)  # clear unpickable attributes
+        return __dict__
+
     def render(self, **kwargs):
         LOGGER.log(self._RENDER_LOG_LEVEL, "Rendering %s", self.id)
 
@@ -214,6 +220,8 @@ class SphinxMapperBase(object):
             if data:
                 self.paths[path] = data
 
+        return True
+
     @staticmethod
     def find_files(patterns, dirs, ignore):
         # pylint: disable=too-many-nested-blocks

autoapi/mappers/dotnet.py

@@ -119,6 +119,8 @@ class DotNetSphinxMapper(SphinxMapperBase):
             if data:
                 self.paths[xdoc_path] = data
 
+        return True
+
     def read_file(self, path, **kwargs):
         """Read file input into memory, returning deserialized objects
 

autoapi/mappers/go.py

@@ -30,6 +30,8 @@ class GoSphinxMapper(SphinxMapperBase):
             if data:
                 self.paths[_dir] = data
 
+        return True
+
     def read_file(self, path, **kwargs):
         """Read file input into memory, returning deserialized objects
 

autoapi/mappers/python/mapper.py

@@ -3,6 +3,7 @@ import copy
 import operator
 import os
 
+import sphinx.environment
 import sphinx.util
 from sphinx.util.console import bold
 import sphinx.util.docstrings
@@ -226,6 +227,22 @@ class PythonSphinxMapper(SphinxMapperBase):
             self.app.config.autoapi_python_use_implicit_namespaces
         )
 
+    def _need_to_load(self, files):
+        last_files = getattr(self.app.env, "autoapi_source_files", [])
+        self.app.env.autoapi_source_files = files
+
+        last_mtime = getattr(self.app.env, "autoapi_max_mtime", 0)
+        this_mtime = max(os.path.getmtime(file) for _, file in files)
+        self.app.env.autoapi_max_mtime = this_mtime
+
+        if not self.app.config.autoapi_keep_files:
+            return True
+
+        if self.app.env.config_status != sphinx.environment.CONFIG_OK:
+            return True
+
+        return last_files != files or not last_mtime or last_mtime < this_mtime
+
     def _find_files(self, patterns, dirs, ignore):
         for dir_ in dirs:
             dir_root = dir_
@@ -245,6 +262,12 @@ class PythonSphinxMapper(SphinxMapperBase):
         shortened, relative path the package/module
         """
         dir_root_files = list(self._find_files(patterns, dirs, ignore))
+        if not self._need_to_load(dir_root_files):
+            LOGGER.debug(
+                "[AutoAPI] Skipping read stage because source files have not changed."
+            )
+            return False
+
         for dir_root, path in sphinx.util.status_iterator(
             dir_root_files,
             bold("[AutoAPI] Reading files... "),
@@ -256,6 +279,8 @@ class PythonSphinxMapper(SphinxMapperBase):
                 data["relative_path"] = os.path.relpath(path, dir_root)
                 self.paths[path] = data
 
+        return True
+
     def read_file(self, path, **kwargs):
         """Read file input into memory, returning deserialized objects
 

docs/reference/config.rst

@@ -229,7 +229,7 @@ The following events allow you to control the behaviour of AutoAPI.
 	:param options: The options given to the directive.
 
 
-Debugging Options
+Advanced Options
 -----------------
 
 .. confval:: autoapi_keep_files
@@ -238,3 +238,7 @@ Debugging Options
 
 	Keep the AutoAPI generated files on the filesystem after the run.
 	Useful for debugging or transitioning to manual documentation.
+
+	Keeping files will also allow AutoAPI to use incremental builds.
+	Providing none of the source files have changed,
+	AutoAPI will skip parsing the source code and regenerating the API documentation.

tests/python/test_pyintegration.py

@@ -19,22 +19,26 @@ from autoapi.mappers.python import (
 )
 
 
+def rebuild(confoverrides=None, **kwargs):
+    app = Sphinx(
+        srcdir=".",
+        confdir=".",
+        outdir="_build/text",
+        doctreedir="_build/.doctrees",
+        buildername="text",
+        confoverrides=confoverrides,
+        **kwargs
+    )
+    app.build()
+
+
 @pytest.fixture(scope="class")
 def builder():
     cwd = os.getcwd()
 
     def build(test_dir, confoverrides=None, **kwargs):
         os.chdir("tests/python/{0}".format(test_dir))
-        app = Sphinx(
-            srcdir=".",
-            confdir=".",
-            outdir="_build/text",
-            doctreedir="_build/.doctrees",
-            buildername="text",
-            confoverrides=confoverrides,
-            **kwargs
-        )
-        app.build(force_all=True)
+        rebuild(confoverrides=confoverrides, **kwargs)
 
     yield build
 
@@ -686,6 +690,51 @@ class TestComplexPackageParallel(object):
         builder("pypackagecomplex", parallel=2)
 
 
+def test_caching(builder):
+    mtimes = (0, 0)
+
+    def record_mtime():
+        nonlocal mtimes
+        mtime = 0
+        for root, _, files in os.walk("_build/text/autoapi"):
+            for name in files:
+                this_mtime = os.path.getmtime(os.path.join(root, name))
+                mtime = max(mtime, this_mtime)
+
+        mtimes = (*mtimes[1:], mtime)
+
+    builder("pypackagecomplex", confoverrides={"autoapi_keep_files": True})
+    record_mtime()
+
+    rebuild(confoverrides={"autoapi_keep_files": True})
+    record_mtime()
+
+    assert mtimes[1] == mtimes[0]
+
+    # Check that adding a file rebuilds the docs
+    extra_file = "complex/new.py"
+    with open(extra_file, "w") as out_f:
+        out_f.write("\n")
+
+    try:
+        rebuild(confoverrides={"autoapi_keep_files": True})
+    finally:
+        os.remove(extra_file)
+
+    record_mtime()
+    assert mtimes[1] != mtimes[0]
+
+    # Removing a file also rebuilds the docs
+    rebuild(confoverrides={"autoapi_keep_files": True})
+    record_mtime()
+    assert mtimes[1] != mtimes[0]
+
+    # Changing not keeping files always builds
+    rebuild()
+    record_mtime()
+    assert mtimes[1] != mtimes[0]
+
+
 class TestImplicitNamespacePackage(object):
     @pytest.fixture(autouse=True, scope="class")
     def built(self, builder):