Have update-bundled check for updates to all bundled code

Add a test for whether we have bundled code inside of ansible that needs to be updated

docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst

@@ -0,0 +1,31 @@
+:orphan:
+
+Sanity Tests » update-bundled
+=============================
+
+Check whether any of our known bundled code needs to be updated for a new upstream release.
+
+This test can error in the following ways:
+
+* The bundled code is out of date with regard to the latest release on pypi.  Update the code
+  to the new version and update the version in _BUNDLED_METADATA to solve this.
+
+* The code is lacking a _BUNDLED_METADATA variable.  This typically happens when a bundled version
+  is updated and we forget to add a _BUNDLED_METADATA variable to the updated file.  Once that is
+  added, this error should go away.
+
+* A file has a _BUNDLED_METADATA variable but the file isn't specified in
+  :file:`test/sanity/code-smell/update-bundled.py`.  This typically happens when a new bundled
+  library is added.  Add the file to the `get_bundled_libs()` function in the `update-bundled.py`
+  test script to solve this error.
+
+_BUNDLED_METADATA has the following fields:
+
+:pypi_name: Name of the bundled package on pypi
+
+:version: Version of the package that we are including here
+
+:version_constraints: Optional PEP440 specifier for the version range that we are bundling.
+                      Currently, the only valid use of this is to follow a version that is
+                      compatible with the Python stdlib when newer versions of the pypi package
+                      implement a new API.

hacking/update_bundled.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-
-import glob
-import json
-import os.path
-from distutils.version import LooseVersion
-
-from ansible.module_utils.urls import open_url
-
-basedir = os.path.dirname(__file__)
-
-for filename in glob.glob(os.path.join(basedir, '../lib/ansible/compat/*/__init__.py')):
-    if 'compat/tests' in filename:
-        # compat/tests doesn't bundle any code
-        continue
-
-    filename = os.path.normpath(filename)
-    with open(filename, 'r') as module:
-        for line in module:
-            if line.strip().startswith('_BUNDLED_METADATA'):
-                data = line[line.index('{'):].strip()
-                break
-        else:
-            print('WARNING: {0} contained no metadata.  Could not check for updates'.format(filename))
-            continue
-        metadata = json.loads(data)
-        pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
-        pypi_data = json.loads(pypi_fh.read().decode('utf-8'))
-        if LooseVersion(metadata['version']) < LooseVersion(pypi_data['info']['version']):
-            print('UPDATE: {0} from {1} to {2} {3}'.format(
-                metadata['pypi_name'],
-                metadata['version'],
-                pypi_data['info']['version'],
-                'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))

lib/ansible/compat/selectors/__init__.py

@@ -24,7 +24,8 @@ Compat selectors library.  Python-3.5 has this builtin.  The selectors2
 package exists on pypi to backport the functionality as far as python-2.6.
 '''
 # The following makes it easier for us to script updates of the bundled code
-_BUNDLED_METADATA = {"pypi_name": "selectors2", "version": "1.1.0"}
+_BUNDLED_METADATA = {"pypi_name": "selectors2", "version": "1.1.0", "version_constraints": ">1.0,<2.0"}
+
 # Added these bugfix commits from 2.1.0:
 # * https://github.com/SethMichaelLarson/selectors2/commit/3bd74f2033363b606e1e849528ccaa76f5067590
 #   Wrap kqueue.control so that timeout is a keyword arg

lib/ansible/module_utils/compat/ipaddress.py

@@ -66,6 +66,11 @@ from __future__ import unicode_literals
 import itertools
 import struct
 
+
+# The following makes it easier for us to script updates of the bundled code and is not part of
+# upstream
+_BUNDLED_METADATA = {"pypi_name": "ipaddress", "version": "1.0.22"}
+
 __version__ = '1.0.22'
 
 # Compatibility functions

lib/ansible/module_utils/six/__init__.py

@@ -33,6 +33,10 @@ import operator
 import sys
 import types
 
+# The following makes it easier for us to script updates of the bundled code.  It is not part of
+# upstream six
+_BUNDLED_METADATA = {"pypi_name": "six", "version": "1.11.0"}
+
 __author__ = "Benjamin Peterson <benjamin@python.org>"
 __version__ = "1.11.0"
 

lib/ansible/module_utils/urls.py

@@ -136,6 +136,10 @@ if not HAS_SSLCONTEXT and HAS_SSL:
         del libssl
 
 
+# The following makes it easier for us to script updates of the bundled backports.ssl_match_hostname
+# The bundled backports.ssl_match_hostname should really be moved into its own file for processing
+_BUNDLED_METADATA = {"pypi_name": "backports.ssl_match_hostname", "version": "3.5.0.1"}
+
 LOADED_VERIFY_LOCATIONS = set()
 
 HAS_MATCH_HOSTNAME = True

test/sanity/code-smell/update-bundled.json

@@ -0,0 +1,7 @@
+{
+ "ignore_changes": true,
+ "extensions": [
+    ".py"
+ ],
+ "output": "path-message"
+}

test/sanity/code-smell/update-bundled.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# (c) 2018, Ansible Project
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import fnmatch
+import json
+import os
+import os.path
+import re
+import sys
+from distutils.version import LooseVersion
+
+import packaging.specifiers
+
+from ansible.module_utils.urls import open_url
+
+
+BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b')
+
+
+def get_bundled_libs(paths):
+    bundled_libs = set()
+    for filename in fnmatch.filter(paths, 'lib/ansible/compat/*/__init__.py'):
+        bundled_libs.add(filename)
+
+    bundled_libs.add('lib/ansible/module_utils/distro/__init__.py')
+    bundled_libs.add('lib/ansible/module_utils/six/__init__.py')
+    bundled_libs.add('lib/ansible/module_utils/compat/ipaddress.py')
+    # backports.ssl_match_hostname should be moved to its own file in the future
+    bundled_libs.add('lib/ansible/module_utils/urls.py')
+
+    return bundled_libs
+
+
+def get_files_with_bundled_metadata(paths):
+    with_metadata = set()
+    for path in paths:
+        if path == 'test/sanity/code-smell/update-bundled.py':
+            continue
+
+        with open(path, 'rb') as f:
+            body = f.read()
+
+        if BUNDLED_RE.search(body):
+            with_metadata.add(path)
+
+    return with_metadata
+
+
+def get_bundled_metadata(filename):
+    with open(filename, 'r') as module:
+        for line in module:
+            if line.strip().startswith('_BUNDLED_METADATA'):
+                data = line[line.index('{'):].strip()
+                break
+        else:
+            raise ValueError('Unable to check bundled library for update.  Please add'
+                             ' _BUNDLED_METADATA dictionary to the library file with'
+                             ' information on pypi name and bundled version.')
+        metadata = json.loads(data)
+    return metadata
+
+
+def get_latest_applicable_version(pypi_data, constraints=None):
+    latest_version = "0"
+    if 'version_constraints' in metadata:
+        version_specification = packaging.specifiers.SpecifierSet(metadata['version_constraints'])
+        for version in pypi_data['releases']:
+            if version in version_specification:
+                if LooseVersion(version) > LooseVersion(latest_version):
+                    latest_version = version
+    else:
+        latest_version = pypi_data['info']['version']
+
+    return latest_version
+
+
+if __name__ == '__main__':
+    paths = sys.argv[1:] or sys.stdin.read().splitlines()
+
+    bundled_libs = get_bundled_libs(paths)
+    files_with_bundled_metadata = get_files_with_bundled_metadata(paths)
+
+    for filename in files_with_bundled_metadata.difference(bundled_libs):
+        print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to'
+              ' test/sanity/code-smell/update-bundled.py'.format(filename))
+
+    for filename in bundled_libs:
+        try:
+            metadata = get_bundled_metadata(filename)
+        except ValueError as e:
+            print('{0}: ERROR: {1}'.format(filename, e))
+            continue
+        except (IOError, OSError) as e:
+            if e.errno == 2:
+                print('{0}: ERROR: {1}.  Perhaps the bundled library has been removed'
+                      ' or moved and the bundled library test needs to be modified as'
+                      ' well?'.format(filename, e))
+
+        pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
+        pypi_data = json.loads(pypi_fh.read().decode('utf-8'))
+
+        constraints = metadata.get('version_constraints', None)
+        latest_version = get_latest_applicable_version(pypi_data, constraints)
+
+        if LooseVersion(metadata['version']) < LooseVersion(latest_version):
+            print('{0}: UPDATE {1} from {2} to {3} {4}'.format(
+                filename,
+                metadata['pypi_name'],
+                metadata['version'],
+                latest_version,
+                'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))