From 9dda393d7036c86e69dd1a4dfbe0f72bf5f9bc5b Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 17 Jul 2020 13:07:35 -0700 Subject: [PATCH] Collections docs generation (#59761) * Build documentation for Ansible-2.10 (formerly known as ACD). Builds plugin docs from collections whose source is on galaxy The new command downloads collections from galaxy, then finds the plugins inside of them to get the documentation for those plugins. * Update the python syntax checks * docs builds can now require python 3.6+. * Move plugin formatter code out to an external tool, antsibull-docs. Collection owners want to be able to extract docs for their own websites as well. * The jinja2 filters, tests, and other support code have moved to antsibull * Remove document_plugins as that has now been integrated into antsibull-docs * Cleanup and bugfix to other build script code: * The Commands class needed to have its metaclass set for abstractmethod to work correctly * Fix lint issues in some command plugins * Add the docs/docsite/rst/collections to .gitignore as everything in that directory will be generated so we don't want any of it saved in the git repository * gitignore the build dir and remove edit docs link on module pages * Add docs/rst/collections as a directory to remove on make clean * Split the collections docs from the main docs * remove version and edit on github * remove version banner for just collections * clarify examples need collection keyword defined * Remove references to plugin documentation locations that no longer exist. * Perhaps the pages in plugins/*.rst should be deprecated altogether and their content moved? * If not, perhaps we want to rephrase and link into the collection documentation? * Or perhaps we want to link to the plugins which are present in collections/ansible/builtin? * Remove PYTHONPATH from the build-ansible calls One of the design goals of the build-ansible.py script was for it to automatically set its library path to include the checkout of ansible and the library of code to implement itself. Because it automatically includes the checkout of ansible, we don't need to set PYTHONPATH in the Makefile any longer. * Create a command to only build ansible-base plugin docs * When building docs for devel, only build the ansible-base docs for now. This is because antsibull needs support for building a "devel tree" of docs. This can be changed once that is implemented * When building docs for the sanity tests, only build the ansible-base plugin docs for now. Those are the docs which are in this repo so that seems appropriate for now. --- .gitignore | 2 + Makefile | 2 +- docs/docsite/Makefile | 76 +- .../sphinx_rtd_theme/ansible_banner.html | 45 +- .../sphinx_rtd_theme/ansible_versions.html | 2 +- docs/docsite/collection-plugins.yml | 17 + docs/docsite/requirements.txt | 1 + docs/docsite/rst/conf.py | 4 + docs/docsite/rst/index.rst | 2 +- docs/docsite/rst/network/index.rst | 2 +- docs/docsite/rst/plugins/become.rst | 5 - docs/docsite/rst/plugins/cache.rst | 5 - docs/docsite/rst/plugins/callback.rst | 6 - docs/docsite/rst/plugins/connection.rst | 6 - docs/docsite/rst/plugins/inventory.rst | 5 - docs/docsite/rst/plugins/lookup.rst | 5 - docs/docsite/rst/plugins/shell.rst | 5 - docs/docsite/rst/plugins/strategy.rst | 5 - docs/docsite/rst/plugins/vars.rst | 5 - docs/docsite/rst/user_guide/become.rst | 2 +- .../rst/user_guide/collections_using.rst | 3 +- docs/docsite/rst/user_guide/modules.rst | 3 +- .../templates/list_of_CATEGORY_modules.rst.j2 | 48 -- .../templates/list_of_CATEGORY_plugins.rst.j2 | 36 - docs/templates/modules_by_support.rst.j2 | 45 - docs/templates/plugin.rst.j2 | 442 ---------- docs/templates/plugin_deprecation_stub.rst.j2 | 18 - docs/templates/plugins_by_category.rst.j2 | 9 - docs/templates/plugins_by_support.rst.j2 | 15 - hacking/build-ansible.py | 17 +- .../command_plugins/collection_meta.py | 10 +- .../command_plugins/docs_build.py | 164 ++++ .../command_plugins/plugin_formatter.py | 807 ------------------ .../build_ansible/jinja2/__init__.py | 0 .../build_ansible/jinja2/filters.py | 100 --- test/sanity/code-smell/docs-build.py | 2 +- .../code-smell/docs-build.requirements.txt | 1 + test/sanity/ignore.txt | 3 - 38 files changed, 285 insertions(+), 1640 deletions(-) create mode 100644 docs/docsite/collection-plugins.yml delete mode 100644 docs/templates/list_of_CATEGORY_modules.rst.j2 delete mode 100644 docs/templates/list_of_CATEGORY_plugins.rst.j2 delete mode 100644 docs/templates/modules_by_support.rst.j2 delete mode 100644 docs/templates/plugin.rst.j2 delete mode 100644 docs/templates/plugin_deprecation_stub.rst.j2 delete mode 100644 docs/templates/plugins_by_category.rst.j2 delete mode 100644 docs/templates/plugins_by_support.rst.j2 create mode 100644 hacking/build_library/build_ansible/command_plugins/docs_build.py delete mode 100644 hacking/build_library/build_ansible/command_plugins/plugin_formatter.py delete mode 100644 hacking/build_library/build_ansible/jinja2/__init__.py delete mode 100644 hacking/build_library/build_ansible/jinja2/filters.py diff --git a/.gitignore b/.gitignore index cdd3941dee0..55909288176 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ docs/docsite/rst/cli/ansible.rst docs/docsite/rst/dev_guide/collections_galaxy_meta.rst docs/docsite/rst/dev_guide/testing/sanity/index.rst.new docs/docsite/rst/modules/*.rst +docs/docsite/rst/collections/*.rst +docs/docsite/rst/collections/*/*.rst docs/docsite/rst/playbooks_directives.rst docs/docsite/rst/plugins_by_category.rst docs/docsite/rst/plugins/*/*.rst diff --git a/Makefile b/Makefile index 79110d8f075..4c504a858a0 100644 --- a/Makefile +++ b/Makefile @@ -277,7 +277,7 @@ linkcheckdocs: .PHONY: generate_rst generate_rst: lib/ansible/cli/*.py mkdir -p ./docs/man/man1/ ; \ - PYTHONPATH=./lib $(GENERATE_CLI) --template-file=docs/templates/man.j2 --output-dir=docs/man/man1/ --output-format man lib/ansible/cli/*.py + $(GENERATE_CLI) --template-file=docs/templates/man.j2 --output-dir=docs/man/man1/ --output-format man lib/ansible/cli/*.py docs: generate_rst diff --git a/docs/docsite/Makefile b/docs/docsite/Makefile index 3b0b512b243..8993da1e3e5 100644 --- a/docs/docsite/Makefile +++ b/docs/docsite/Makefile @@ -1,6 +1,6 @@ OS := $(shell uname -s) SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"): -PLUGIN_FORMATTER=../../hacking/build-ansible.py document-plugins +PLUGIN_FORMATTER=../../hacking/build-ansible.py docs-build TESTING_FORMATTER=../bin/testing_formatter.sh KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords CONFIG_DUMPER=../../hacking/build-ansible.py document-config @@ -12,23 +12,35 @@ else CPUS ?= $(shell nproc) endif -# Sets the build output directory if it's not already specified +# Sets the build output directory for the main docsite if it's not already specified ifndef BUILDDIR BUILDDIR = _build endif -MODULE_ARGS= -ifdef MODULES - MODULE_ARGS = -l $(MODULES) -endif - +# Backwards compat for separate VARS PLUGIN_ARGS= +ifdef MODULES +ifndef PLUGINS + PLUGIN_ARGS = -l $(MODULES) +else + PLUGIN_ARGS = -l $(MODULES),$(PLUGINS) +endif +else ifdef PLUGINS PLUGIN_ARGS = -l $(PLUGINS) endif +endif + DOC_PLUGINS ?= become cache callback cliconf connection httpapi inventory lookup netconf shell strategy vars +PYTHON=python +# fetch version from project release.py as single source-of-truth +VERSION := $(shell $(PYTHON) ../../packaging/release/versionhelper/version_helper.py --raw || echo error) +ifeq ($(findstring error,$(VERSION)), error) +$(error "version_helper failed") +endif + assertrst: ifndef rst $(error specify document or pattern with rst=somefile.rst) @@ -38,17 +50,24 @@ all: docs docs: htmldocs -generate_rst: collections_meta config cli keywords modules plugins testing +generate_rst: collections_meta config cli keywords plugins testing +base_generate_rst: collections_meta config cli keywords base_plugins testing htmldocs: generate_rst CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx html +base_htmldocs: base_generate_rst + CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx html + singlehtmldocs: generate_rst CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx singlehtml +base_singlehtmldocs: base_generate_rst + CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx singlehtml + linkcheckdocs: generate_rst CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx linkcheck - + webdocs: docs #TODO: leaving htmlout removal for those having older versions, should eventually be removed also @@ -58,7 +77,7 @@ clean: -rm -rf $(BUILDDIR)/html -rm -rf htmlout -rm -rf module_docs - -rm -rf _build + -rm -rf $(BUILDDIR) -rm -f .buildinfo -rm -f objects.inv -rm -rf *.doctrees @@ -70,43 +89,44 @@ clean: find . -type f \( -name "*~" -or -name "#*" \) -delete find . -type f \( -name "*.swp" \) -delete @echo "Cleaning up generated rst" - rm -f rst/modules/*_by_category.rst - rm -f rst/modules/list_of_*.rst - rm -f rst/modules/*_maintained.rst - rm -f rst/modules/*_module.rst - rm -f rst/modules/*_plugin.rst rm -f rst/playbooks_directives.rst - rm -f rst/plugins/*/*.rst rm -f rst/reference_appendices/config.rst rm -f rst/reference_appendices/playbooks_keywords.rst rm -f rst/dev_guide/collections_galaxy_meta.rst rm -f rst/cli/*.rst + rm -rf rst/collections/* + @echo "Cleaning up legacy generated rst locations" + rm -rf rst/modules + rm -f rst/plugins/*/*.rst .PHONY: docs clean collections_meta: ../templates/collections_galaxy_meta.rst.j2 - PYTHONPATH=../../lib $(COLLECTION_DUMPER) --template-file=../templates/collections_galaxy_meta.rst.j2 --output-dir=rst/dev_guide/ ../../lib/ansible/galaxy/data/collections_galaxy_meta.yml + $(COLLECTION_DUMPER) --template-file=../templates/collections_galaxy_meta.rst.j2 --output-dir=rst/dev_guide/ ../../lib/ansible/galaxy/data/collections_galaxy_meta.yml # TODO: make generate_man output dir cli option cli: mkdir -p rst/cli - PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py + $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py keywords: ../templates/playbooks_keywords.rst.j2 - PYTHONPATH=../../lib $(KEYWORD_DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ ./keyword_desc.yml + $(KEYWORD_DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ ./keyword_desc.yml config: ../templates/config.rst.j2 - PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ ../../lib/ansible/config/base.yml + $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ ../../lib/ansible/config/base.yml -modules: ../templates/plugin.rst.j2 - PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS) +# For now, if we're building on devel, just build base docs. In the future we'll want to build docs that +# are the latest versions on galaxy (using a different antsibull-docs subcommand) +plugins: + if expr "$(VERSION)" : '.*[.]dev[0-9]\+$$' &> /dev/null; then \ + $(PLUGIN_FORMATTER) base -o rst $(PLUGIN_ARGS);\ + else \ + $(PLUGIN_FORMATTER) full -o rst $(PLUGIN_ARGS);\ + fi -plugins: ../templates/plugin.rst.j2 - @echo "looping over doc plugins" - for plugin in $(DOC_PLUGINS); \ - do \ - PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \ - done +# This only builds the plugin docs included with ansible-base +base_plugins: + $(PLUGIN_FORMATTER) base -o rst $(PLUGIN_ARGS);\ testing: $(TESTING_FORMATTER) diff --git a/docs/docsite/_themes/sphinx_rtd_theme/ansible_banner.html b/docs/docsite/_themes/sphinx_rtd_theme/ansible_banner.html index 7ca2bd4a0fd..ba889e7186e 100644 --- a/docs/docsite/_themes/sphinx_rtd_theme/ansible_banner.html +++ b/docs/docsite/_themes/sphinx_rtd_theme/ansible_banner.html @@ -10,26 +10,27 @@ element.appendChild(para); document.write(''); } - - // Create a banner if we're not the latest version - current_url = window.location.href; - if ((current_url.search("latest") > -1) || (current_url.search("/{{ latest_version }}/") > -1)) { - // no banner for latest release - } else if (current_url.search("devel") > -1) { - document.write(''); - } else { - document.write(''); - } + {% if (not READTHEDOCS) and (available_versions is defined) %} + // Create a banner if we're not the latest version + current_url = window.location.href; + if ((current_url.search("latest") > -1) || (current_url.search("/{{ latest_version }}/") > -1)) { + // no banner for latest release + } else if (current_url.search("devel") > -1) { + document.write(''); + } else { + document.write(''); + } + {% endif %} diff --git a/docs/docsite/_themes/sphinx_rtd_theme/ansible_versions.html b/docs/docsite/_themes/sphinx_rtd_theme/ansible_versions.html index 06a0271eda0..3d3d7fc682d 100644 --- a/docs/docsite/_themes/sphinx_rtd_theme/ansible_versions.html +++ b/docs/docsite/_themes/sphinx_rtd_theme/ansible_versions.html @@ -1,7 +1,7 @@ {# Creates dropdown version selection in the top-left navigation. #}
- {% if not READTHEDOCS %} + {% if (not READTHEDOCS) and (available_versions is defined) %}