Docs: Make docsite rebuilds smarter/faster (#45062)

* Make the following scripts  idempotent so that we only have to rebuild changed docs, not all docs:

  * plugin_formatter
  * generate_man
  * dump_keywords.py
  * dump_config.py
  * testing_formatter.sh

docs/bin/dump_config.py

@@ -6,6 +6,8 @@ import sys
 import yaml
 
 from jinja2 import Environment, FileSystemLoader
+from ansible.module_utils._text import to_bytes
+from ansible.utils._build_helpers import update_file_if_different
 
 DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
 
@@ -62,8 +64,8 @@ def main(args):
     output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
     temp_vars = {'config_options': config_options}
 
-    with open(output_name, 'wb') as f:
-        f.write(template.render(temp_vars).encode('utf-8'))
+    data = to_bytes(template.render(temp_vars))
+    update_file_if_different(output_name, data)
 
     return 0
 

docs/bin/dump_keywords.py

@@ -8,10 +8,12 @@ import jinja2
 import yaml
 from jinja2 import Environment, FileSystemLoader
 
+from ansible.module_utils._text import to_bytes
 from ansible.playbook import Play
 from ansible.playbook.block import Block
 from ansible.playbook.role import Role
 from ansible.playbook.task import Task
+from ansible.utils._build_helpers import update_file_if_different
 
 template_file = 'playbooks_keywords.rst.j2'
 oblist = {}
@@ -79,5 +81,4 @@ if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
     # jinja2 < 2.10's indent filter indents blank lines.  Cleanup
     keyword_page = re.sub(' +\n', '\n', keyword_page)
 
-with open(outputname, 'w') as f:
-    f.write(keyword_page)
+update_file_if_different(outputname, to_bytes(keyword_page))

docs/bin/generate_man.py

@@ -2,12 +2,12 @@
 
 import optparse
 import os
-import pprint
 import sys
 
 from jinja2 import Environment, FileSystemLoader
 
 from ansible.module_utils._text import to_bytes
+from ansible.utils._build_helpers import update_file_if_different
 
 
 def generate_parser():
@@ -274,7 +274,4 @@ if __name__ == '__main__':
 
         manpage = template.render(tvars)
         filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name'])
-
-        with open(filename, 'wb') as f:
-            f.write(to_bytes(manpage))
-            print("Wrote doc to %s" % filename)
+        update_file_if_different(filename, to_bytes(manpage))

docs/bin/plugin_formatter.py

@@ -56,6 +56,7 @@ from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.loader import fragment_loader
 from ansible.utils import plugin_docs
 from ansible.utils.display import Display
+from ansible.utils._build_helpers import update_file_if_different
 
 
 #####################################################################################
@@ -183,8 +184,8 @@ def write_data(text, output_dir, outputname, module=None):
             os.makedirs(output_dir)
         fname = os.path.join(output_dir, outputname)
         fname = fname.replace(".py", "")
-        with open(fname, 'wb') as f:
-            f.write(to_bytes(text))
+
+        update_file_if_different(fname, to_bytes(text))
     else:
         print(text)
 

docs/bin/testing_formatter.sh

@@ -1,6 +1,8 @@
 #!/bin/bash -eu
 
-cat <<- EOF > ../docsite/rst/dev_guide/testing/sanity/index.rst
+FILENAME=../docsite/rst/dev_guide/testing/sanity/index.rst
+
+cat <<- EOF >$FILENAME.new
 Sanity Tests
 ============
 
@@ -12,5 +14,9 @@ This list is also available using \`\`ansible-test sanity --list-tests\`\`.
 
 $(for test in $(../../test/runner/ansible-test sanity --list-tests); do echo "   ${test}"; done)
 
-
 EOF
+
+# Put file into place if it has changed
+if [ "$(sha1sum <$FILENAME)" != "$(sha1sum <$FILENAME.new)" ]; then
+    mv -f $FILENAME.new $FILENAME
+fi

docs/docsite/Makefile

@@ -35,7 +35,7 @@ endif
 
 all: docs
 
-docs: clean htmldocs
+docs: htmldocs
 
 generate_rst: staticmin config cli keywords modules plugins testing
 

lib/ansible/utils/_build_helpers.py

@@ -0,0 +1,38 @@
+# Copyright: (c) 2018, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+"""
+This file contains common code for building ansible. If you want to use code from here at runtime,
+it needs to be moved out of this file and the implementation looked over to figure out whether API
+should be changed before being made public.
+"""
+
+import os.path
+
+
+def update_file_if_different(filename, b_data):
+    '''
+    Replace file content only if content is different.
+
+    This preserves timestamps in case the file content has not changed.  It performs multiple
+    operations on the file so it is not atomic and may be slower than simply writing to the file.
+
+    :arg filename: The filename to write to
+    :b_data: Byte string containing the data to write to the file
+    '''
+    try:
+        with open(filename, 'rb') as f:
+            b_data_old = f.read()
+    except IOError as e:
+        if e.errno != 2:
+            raise
+        # File did not exist, set b_data_old to a sentinel value so that
+        # b_data gets written to the filename
+        b_data_old = None
+
+    if b_data_old != b_data:
+        with open(filename, 'wb') as f:
+            f.write(b_data)

lib/ansible/utils/plugin_docs.py

@@ -1,22 +1,6 @@
-# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
+# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type