Merge pull request #87 from rtfd/toc-addition-extension

Build initial toctree insertion implementation

autoapi/backends.py

@@ -7,11 +7,13 @@ default_file_mapping = {
     'javascript': ['*.js'],
 }
 
+
 default_ignore_patterns = {
     'dotnet': ['*toc.yml', '*index.yml'],
     'python': ['*migrations*'],
 }
 
+
 default_backend_mapping = {
     'python': PythonSphinxMapper,
     'dotnet': DotNetSphinxMapper,

autoapi/extension.py

@@ -15,6 +15,7 @@ from docutils.parsers.rst import directives
 from .backends import default_file_mapping, default_ignore_patterns, default_backend_mapping
 from .directives import NestedParse
 from .settings import API_ROOT
+from .toctree import add_domain_to_toctree
 
 default_options = ['members', 'undoc-members', 'private-members', 'special-members']
 
@@ -54,9 +55,9 @@ def run_autoapi(app):
 
     app.env.autoapi_data = []
 
-    domain = default_backend_mapping[app.config.autoapi_type]
-    domain_obj = domain(app, template_dir=app.config.autoapi_template_dir,
-                        url_root=url_root)
+    sphinx_mapper = default_backend_mapping[app.config.autoapi_type]
+    sphinx_mapper_obj = sphinx_mapper(app, template_dir=app.config.autoapi_template_dir,
+                                      url_root=url_root)
 
     if app.config.autoapi_file_patterns:
         file_patterns = app.config.autoapi_file_patterns
@@ -78,17 +79,17 @@ def run_autoapi(app):
 
     # Actual meat of the run.
     app.info(bold('[AutoAPI] ') + darkgreen('Loading Data'))
-    domain_obj.load(
+    sphinx_mapper_obj.load(
         patterns=file_patterns,
         dirs=normalized_dirs,
         ignore=ignore_patterns,
     )
 
     app.info(bold('[AutoAPI] ') + darkgreen('Mapping Data'))
-    domain_obj.map(options=app.config.autoapi_options)
+    sphinx_mapper_obj.map(options=app.config.autoapi_options)
 
     app.info(bold('[AutoAPI] ') + darkgreen('Rendering Data'))
-    domain_obj.output_rst(
+    sphinx_mapper_obj.output_rst(
         root=normalized_root,
         source_suffix=out_suffix,
     )
@@ -101,21 +102,27 @@ def build_finished(app, exception):
             app.info(bold('[AutoAPI] ') + darkgreen('Cleaning generated .rst files'))
         shutil.rmtree(normalized_root)
 
-        mapper = default_backend_mapping[app.config.autoapi_type]
-        if hasattr(mapper, 'build_finished'):
-            mapper.build_finished(app, exception)
+        sphinx_mapper = default_backend_mapping[app.config.autoapi_type]
+        if hasattr(sphinx_mapper, 'build_finished'):
+            sphinx_mapper.build_finished(app, exception)
 
 
 def doctree_read(app, doctree):
+    """
+    Inject AutoAPI into the TOC Tree dynamically.
+    """
+
     all_docs = set()
     insert = True
     if app.env.docname == 'index':
         nodes = doctree.traverse(toctree)
+        toc_entry = '%s/index' % app.config.autoapi_root
         if not nodes:
             return
         for node in nodes:
             for entry in node['entries']:
                 all_docs.add(entry[1])
+        # Don't insert if it's already present
         for doc in all_docs:
             if doc.find(app.config.autoapi_root) != -1:
                 insert = False
@@ -124,13 +131,17 @@ def doctree_read(app, doctree):
                 (None, u'%s/index' % app.config.autoapi_root)
             )
             nodes[-1]['includefiles'].append(u'%s/index' % app.config.autoapi_root)
-            app.info(bold('[AutoAPI] ') + darkgreen('Adding AutoAPI TOCTree to index.rst'))
+            app.info(bold('[AutoAPI] ') +
+                     darkgreen('Adding AutoAPI TOCTree [%s] to index.rst' % toc_entry)
+                     )
+            app.env.build_toc_from(app.env.docname, doctree)
 
 
 def setup(app):
     app.connect('builder-inited', run_autoapi)
     app.connect('build-finished', build_finished)
     app.connect('doctree-read', doctree_read)
+    app.connect('doctree-resolved', add_domain_to_toctree)
     app.add_config_value('autoapi_type', 'python', 'html')
     app.add_config_value('autoapi_root', API_ROOT, 'html')
     app.add_config_value('autoapi_ignore', [], 'html')

autoapi/mappers/python.py

@@ -1,6 +1,5 @@
 import sys
 import os
-import re
 import textwrap
 import ast
 from collections import defaultdict
@@ -174,8 +173,9 @@ class PythonPythonMapper(PythonMapperBase):
         # exceptions, including SyntaxError
         try:
             parsed = ast.parse(source)
-        except:  # noqa
-            return
+        except Exception as e:  # noqa
+            print("Error parsing AST: %s" % str(e))
+            return []
         parsed_args = parsed.body[0].args
         arg_names = [arg.id if sys.version_info < (3,) else arg.arg
                      for arg in parsed_args.args]

autoapi/templates/python/class.rst

@@ -1,7 +1,3 @@
-.. autoapi-hidden::
-   {{ obj.short_name }}
-   {{ "=" * obj.short_name|length }}
-
 .. py:class:: {{ obj.short_name }}{% if obj.args %}({{ obj.args|join(',') }}){% endif %}
 
    {%- if obj.docstring %}

autoapi/toctree.py

@@ -0,0 +1,140 @@
+"""
+A small Sphinx extension that adds Domain objects (eg. Python Classes & Methods) to the TOC Tree.
+
+It dynamically adds them to the already rendered ``app.env.tocs`` dict on the Sphinx environment.
+Traditionally this only contains Section's,
+we then nest our Domain references inside the already existing Sections.
+"""
+
+from docutils import nodes
+from sphinx import addnodes
+
+
+def _build_toc_node(docname, anchor='anchor', text='test text', bullet=False):
+    """
+    Create the node structure that Sphinx expects for TOC Tree entries.
+
+    The ``bullet`` argument wraps it in a ``nodes.bullet_list``,
+    which is how you nest TOC Tree entries.
+    """
+    reference = nodes.reference('', '', internal=True, refuri=docname,
+                                anchorname='#' + anchor, *[nodes.Text(text, text)])
+    para = addnodes.compact_paragraph('', '', reference)
+    ret_list = nodes.list_item('', para)
+    if not bullet:
+        return ret_list
+    else:
+        return nodes.bullet_list('', ret_list)
+
+
+def _traverse_parent(node, objtypes):
+    """
+    Traverse up the node's parents until you hit the ``objtypes`` referenced.
+
+    Can either be a single type,
+    or a tuple of types.
+    """
+    curr_node = node.parent
+    while curr_node is not None:
+        if isinstance(curr_node, objtypes):
+            return curr_node
+        curr_node = curr_node.parent
+    return None
+
+
+def _find_toc_node(toc, ref_id, objtype):
+    """
+    Find the actual TOC node for a ref_id.
+
+    Depends on the object type:
+    * Section - First section (refuri) or 2nd+ level section (anchorname)
+    * Desc - Just use the anchor name
+    """
+    for check_node in toc.traverse(nodes.reference):
+        if objtype == nodes.section and \
+                (check_node.attributes['refuri'] == ref_id or
+                 check_node.attributes['anchorname'] == '#' + ref_id):
+            return check_node
+        if objtype == addnodes.desc and check_node.attributes['anchorname'] == '#' + ref_id:
+            return check_node
+    return None
+
+
+def _get_toc_reference(app, node, toc, docname):
+    """
+    Logic that understands maps a specific node to it's part of the toctree.
+
+    It takes a specific incoming ``node``,
+    and returns the actual TOC Tree node that is said reference.
+    """
+    if isinstance(node, nodes.section) and \
+            isinstance(node.parent, nodes.document):
+        # Top Level Section header
+        ref_id = docname
+        toc_reference = _find_toc_node(toc, ref_id, nodes.section)
+    elif isinstance(node, nodes.section):
+        # Nested Section header
+        ref_id = node.attributes['ids'][0]
+        toc_reference = _find_toc_node(toc, ref_id, nodes.section)
+    else:
+        # Desc node
+        try:
+            ref_id = node.children[0].attributes['ids'][0]
+            toc_reference = _find_toc_node(toc, ref_id, addnodes.desc)
+        except (KeyError, IndexError) as e:
+            app.warn('Invalid desc node: %s' % e)
+            toc_reference = None
+
+    return toc_reference
+
+
+def add_domain_to_toctree(app, doctree, docname):
+    """
+    Add domain objects to the toctree dynamically.
+
+    This should be attached to the ``doctree-resolved`` event.
+    This works by:
+
+    * Finding each domain node (addnodes.desc)
+    * Figuring out it's parent that will be in the toctree
+         (nodes.section, or a previously added addnodes.desc)
+    * Finding that parent in the TOC Tree based on it's ID
+    * Taking that element in the TOC Tree,
+      and finding it's parent that is a TOC Listing (nodes.bullet_list)
+    * Adding the new TOC element for our specific node as a child of that nodes.bullet_list
+        * This checks that bullet_list's last child,
+        and checks that it is also a nodes.bullet_list,
+        effectively nesting it under that element
+    """
+    toc = app.env.tocs[docname]
+    for desc_node in doctree.traverse(addnodes.desc):
+        try:
+            ref_id = desc_node.children[0].attributes['ids'][0]
+        except (KeyError, IndexError) as e:
+            app.warn('Invalid desc node: %s' % e)
+            continue
+        try:
+            # Python domain object
+            ref_text = desc_node[0].attributes['fullname'].split('.')[-1].split('(')[0]
+        except (KeyError, IndexError):
+            # TODO[eric]: Support other Domains and ways of accessing this data
+            # Use `astext` for other types of domain objects
+            ref_text = desc_node[0].astext().split('.')[-1].split('(')[0]
+        # This is the actual object that will exist in the TOC Tree
+        # Sections by default, and other Desc nodes that we've previously placed.
+        parent_node = _traverse_parent(node=desc_node, objtypes=(addnodes.desc, nodes.section))
+
+        if parent_node:
+            toc_reference = _get_toc_reference(app, parent_node, toc, docname)
+            if toc_reference:
+                # Get the last child of our parent's bullet list, this is where "we" live.
+                toc_insertion_point = _traverse_parent(toc_reference, nodes.bullet_list)[-1]
+                # Ensure we're added another bullet list so that we nest inside the parent,
+                # not next to it
+                if toc_insertion_point and isinstance(toc_insertion_point[0], nodes.bullet_list):
+                    new_insert = toc_insertion_point[0]
+                    to_add = _build_toc_node(docname, anchor=ref_id, text=ref_text)
+                    new_insert.append(to_add)
+                else:
+                    to_add = _build_toc_node(docname, anchor=ref_id, text=ref_text, bullet=True)
+                    toc_insertion_point.append(to_add)

tests/test_integration.py

@@ -134,3 +134,13 @@ class TOCTreeTests(LanguageIntegrationTests):
             '_build/text/index.txt',
             'AutoAPI Index'
         )
+
+    def test_toctree_domain_insertion(self):
+        """
+        Test that the example_function gets added to the TOC Tree
+        """
+        self._run_test(
+            'toctreeexample',
+            '_build/text/index.txt',
+            '* example_function'
+        )