From 3161a91d7e4c8ae16bdc0563edf66d71e8424e69 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 1 May 2019 13:57:03 -0500 Subject: [PATCH] Implement a framework for having common code for release scripts (#55893) * Implement a framework for having common code for release scripts * Release scripts will go through hacking/build-ansible. build-ansible is a pluggable script which will set a directory that has common code for non-enduser scripts. It will then invoke the plugin which implements that subcommand. Uses straight.plugin for loading each sub-command. * We're going to add tools which are needed to test ansible (the changelog generation, for instance) so we need to include the pieces relevant to that in the tarball. * Add straight.plugin to the sanity test requirements for the same reason * Skip compile test just for build-ansible plugins which won't be run as part of sanity tests. --- MANIFEST.in | 2 + hacking/build-ansible | 76 +++++++++++++++++++ hacking/build_library/__init__.py | 0 .../build_library/build_ansible/__init__.py | 0 .../command_plugins/porting_guide.py} | 35 ++++----- .../command_plugins/release_announcement.py} | 59 +++++++------- .../build_library/build_ansible/commands.py | 50 ++++++++++++ test/runner/requirements/sanity.txt | 1 + test/sanity/code-smell/shebang.py | 3 +- test/sanity/compile/python2.6-skip.txt | 4 +- test/sanity/compile/python2.7-skip.txt | 4 +- test/sanity/compile/python3.5-skip.txt | 4 +- 12 files changed, 185 insertions(+), 53 deletions(-) create mode 100755 hacking/build-ansible create mode 100644 hacking/build_library/__init__.py create mode 100644 hacking/build_library/build_ansible/__init__.py rename hacking/{porting-guide.py => build_library/build_ansible/command_plugins/porting_guide.py} (78%) mode change 100755 => 100644 rename hacking/{release-announcement.py => build_library/build_ansible/command_plugins/release_announcement.py} (82%) mode change 100755 => 100644 create mode 100644 hacking/build_library/build_ansible/commands.py diff --git a/MANIFEST.in b/MANIFEST.in index 2bb21538e0..f70bced34a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,3 +25,5 @@ include changelogs/CHANGELOG*.rst include contrib/README.md recursive-include contrib/inventory * exclude test/sanity/code-smell/botmeta.* +recursive-include hacking/build_library *.py +include hacking/build-ansible diff --git a/hacking/build-ansible b/hacking/build-ansible new file mode 100755 index 0000000000..69f98c6f48 --- /dev/null +++ b/hacking/build-ansible @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# coding: utf-8 +# PYTHON_ARGCOMPLETE_OK +# Copyright: (c) 2019, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +import argparse +import os.path +import sys + +from straight.plugin import load + +try: + import argcomplete +except ImportError: + argcomplete = None + + +def set_sys_path(this_script=__file__): + """Add path to the common librarydirectory to :attr:`sys.path`""" + hacking_dir = os.path.dirname(this_script) + libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library')) + + if libdir not in sys.path: + sys.path.insert(0, libdir) + + +set_sys_path() + +from build_ansible import commands + + +def create_arg_parser(program_name): + """ + Creates a command line argument parser + + :arg program_name: The name of the script. Used in help texts + """ + parser = argparse.ArgumentParser(prog=program_name, + description="Implements utilities to build Ansible") + return parser + + +def main(): + """ + Main entrypoint of the script + + "It all starts here" + """ + subcommands = load('build_ansible.command_plugins', subclasses=commands.Command) + + arg_parser = create_arg_parser(os.path.basename(sys.argv[0])) + subparsers = arg_parser.add_subparsers(title='Subcommands', dest='command', + help='for help use build-ansible SUBCOMMANDS -h') + subcommands.pipe('init_parser', subparsers.add_parser) + + if argcomplete: + argcomplete.autocomplete(arg_parser) + + args = arg_parser.parse_args(sys.argv[1:]) + + for subcommand in subcommands: + if subcommand.name == args.command: + sys.exit(subcommand.main(args)) + + print('Error: Select a subcommand') + arg_parser.print_usage() + + +if __name__ == '__main__': + main() diff --git a/hacking/build_library/__init__.py b/hacking/build_library/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hacking/build_library/build_ansible/__init__.py b/hacking/build_library/build_ansible/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hacking/porting-guide.py b/hacking/build_library/build_ansible/command_plugins/porting_guide.py old mode 100755 new mode 100644 similarity index 78% rename from hacking/porting-guide.py rename to hacking/build_library/build_ansible/command_plugins/porting_guide.py index 61576f3455..efdb674f1c --- a/hacking/porting-guide.py +++ b/hacking/build_library/build_ansible/command_plugins/porting_guide.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # coding: utf-8 # Copyright: (c) 2019, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) @@ -9,10 +8,14 @@ __metaclass__ = type import argparse +import os.path import sys from jinja2 import Environment, DictLoader +# Pylint doesn't understand Python3 namespace modules. +from ..commands import Command # pylint: disable=relative-beyond-top-level + PORTING_GUIDE_TEMPLATE = """ .. _porting_{{ ver }}_guide: @@ -106,16 +109,6 @@ JINJA_ENV = Environment( ) -def parse_args(args): - parser = argparse.ArgumentParser(description="Generate a fresh porting guide template") - parser.add_argument("--version", dest="version", type=str, required=True, action='store', - help="Version of Ansible to write the porting guide for") - - args = parser.parse_args(args) - - return args - - def generate_porting_guide(version): template = JINJA_ENV.get_template('porting_guide') @@ -133,13 +126,17 @@ def write_guide(version, guide_content): out_file.write(guide_content) -def main(): - args = parse_args(sys.argv[1:]) +class PortingGuideCommand(Command): + name = 'porting-guide' - guide_content = generate_porting_guide(args.version) + @classmethod + def init_parser(cls, add_parser): + parser = add_parser(cls.name, description="Generate a fresh porting guide template") + parser.add_argument("--version", dest="version", type=str, required=True, action='store', + help="Version of Ansible to write the porting guide for") - write_guide(args.version, guide_content) - - -if __name__ == '__main__': - main() + @staticmethod + def main(args): + guide_content = generate_porting_guide(args.version) + write_guide(args.version, guide_content) + return 0 diff --git a/hacking/release-announcement.py b/hacking/build_library/build_ansible/command_plugins/release_announcement.py old mode 100755 new mode 100644 similarity index 82% rename from hacking/release-announcement.py rename to hacking/build_library/build_ansible/command_plugins/release_announcement.py index cea3845a6c..a0ff09b482 --- a/hacking/release-announcement.py +++ b/hacking/build_library/build_ansible/command_plugins/release_announcement.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # coding: utf-8 # Copyright: (c) 2019, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) @@ -12,6 +11,7 @@ import argparse import asyncio import datetime import hashlib +import os.path import sys from collections import UserString from distutils.version import LooseVersion @@ -19,6 +19,10 @@ from distutils.version import LooseVersion import aiohttp from jinja2 import Environment, DictLoader +# Pylint doesn't understand Python3 namespace modules. +from ..commands import Command # pylint: disable=relative-beyond-top-level + + # pylint: disable= VERSION_FRAGMENT = """ {%- if versions | length > 1 %} @@ -31,7 +35,7 @@ VERSION_FRAGMENT = """ """ LONG_TEMPLATE = """ -{% set plural = True if versions | length == 1 else False %} +{% set plural = False if versions | length == 1 else True %} {% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %} To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com @@ -66,7 +70,7 @@ What's new in {{ version_str }} {{ '-' * (14 + version_str | length) }} {% filter wordwrap %} -{% if plural %}This release is a{% else %}These releases are{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if versions | length <= 1 %} changelog is{% else %} changelogs are{% endif %} at: +{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at: {% endfilter %} @@ -116,15 +120,16 @@ Thanks! # proper wrapping to occur SHORT_TEMPLATE = """ +{% set plural = False if versions | length == 1 else True %} @ansible {{ version_str }} -{% if versions | length > 1 %} +{% if plural %} have {% else %} has {% endif %} been released! Get -{% if versions | length > 1 %} +{% if plural %} them {% else %} it @@ -152,19 +157,7 @@ class VersionStr(UserString): self.ver_obj = LooseVersion(string) -def parse_args(args): - parser = argparse.ArgumentParser(description="Generate email and twitter announcements" - " from template") - parser.add_argument("--version", dest="versions", type=str, required=True, action='append', - help="Versions of Ansible to announce") - parser.add_argument("--name", type=str, required=True, help="Real name to use on emails") - parser.add_argument("--email-out", type=str, default="-", - help="Filename to place the email announcement into") - parser.add_argument("--twitter-out", type=str, default="-", - help="Filename to place the twitter announcement into") - - args = parser.parse_args(args) - +def transform_args(args): # Make it possible to sort versions in the jinja2 templates new_versions = [] for version in args.versions: @@ -285,15 +278,29 @@ def write_message(filename, message): sys.stdout.write(message) -def main(): - args = parse_args(sys.argv[1:]) +class ReleaseAnnouncementCommand(Command): + name = 'release-announcement' - twitter_message = generate_short_message(args.versions) - email_message = generate_long_message(args.versions, args.name) + @classmethod + def init_parser(cls, add_parser): + parser = add_parser(cls.name, + description="Generate email and twitter announcements from template") - write_message(args.twitter_out, twitter_message) - write_message(args.email_out, email_message) + parser.add_argument("--version", dest="versions", type=str, required=True, action='append', + help="Versions of Ansible to announce") + parser.add_argument("--name", type=str, required=True, help="Real name to use on emails") + parser.add_argument("--email-out", type=str, default="-", + help="Filename to place the email announcement into") + parser.add_argument("--twitter-out", type=str, default="-", + help="Filename to place the twitter announcement into") + @staticmethod + def main(args): + args = transform_args(args) -if __name__ == '__main__': - main() + twitter_message = generate_short_message(args.versions) + email_message = generate_long_message(args.versions, args.name) + + write_message(args.twitter_out, twitter_message) + write_message(args.email_out, email_message) + return 0 diff --git a/hacking/build_library/build_ansible/commands.py b/hacking/build_library/build_ansible/commands.py new file mode 100644 index 0000000000..363e8056fc --- /dev/null +++ b/hacking/build_library/build_ansible/commands.py @@ -0,0 +1,50 @@ +# coding: utf-8 +# Copyright: (c) 2019, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +from abc import ABCMeta, abstractmethod, abstractproperty + + +class Command: + """ + Subcommands of :program:`build-ansible`. + + This defines an interface that all subcommands must conform to. :program:`build-ansible` will + require that these things are present in order to proceed. + """ + @staticmethod + @abstractproperty + def name(): + """Name of the command. The same as the string is invoked with""" + + @staticmethod + @abstractmethod + def init_parser(add_parser): + """ + Initialize and register an argparse ArgumentParser + + :arg add_parser: function which creates an ArgumentParser for the main program. + + Implementations should first create an ArgumentParser using `add_parser` and then populate + it with the command line arguments that are needed. + + .. seealso: + `add_parser` information in the :py:meth:`ArgumentParser.add_subparsers` documentation. + """ + + @staticmethod + @abstractmethod + def main(arguments): + """ + Run the command + + :arg arguments: The **parsed** command line args + + This is the Command's entrypoint. The command line args are already parsed but from here + on, the command can do its work. + """ diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt index 17c8dacbfb..7eb65efa65 100644 --- a/test/runner/requirements/sanity.txt +++ b/test/runner/requirements/sanity.txt @@ -7,6 +7,7 @@ pylint ; python_version >= '3.5' # pylint 2.0.0 and later require python 3+ pytest rstcheck ; python_version >= '2.7' # rstcheck requires python 2.7+ sphinx +straight.plugin # needed for hacking/build-ansible which will host changelog generation virtualenv voluptuous yamllint diff --git a/test/sanity/code-smell/shebang.py b/test/sanity/code-smell/shebang.py index 4d00f80bf5..21e006eeda 100755 --- a/test/sanity/code-smell/shebang.py +++ b/test/sanity/code-smell/shebang.py @@ -39,8 +39,7 @@ def main(): 'test/utils/shippable/timing.py', 'test/integration/targets/old_style_modules_posix/library/helloworld.sh', # The following are Python 3.6+. Only run by release engineers - 'hacking/release-announcement.py', - 'hacking/porting-guide.py', + 'hacking/build-ansible', ]) # see https://unicode.org/faq/utf_bom.html#bom1 diff --git a/test/sanity/compile/python2.6-skip.txt b/test/sanity/compile/python2.6-skip.txt index 60a60b5c76..5814217d4a 100644 --- a/test/sanity/compile/python2.6-skip.txt +++ b/test/sanity/compile/python2.6-skip.txt @@ -1,3 +1,3 @@ # The following are only run by release engineers who can be asked to have newer Python3 on their systems -hacking/release-announcement.py -hacking/porting-guide.py +hacking/build_library/build_ansible/command_plugins/porting_guide.py +hacking/build_library/build_ansible/command_plugins/release_announcement.py diff --git a/test/sanity/compile/python2.7-skip.txt b/test/sanity/compile/python2.7-skip.txt index 60a60b5c76..5814217d4a 100644 --- a/test/sanity/compile/python2.7-skip.txt +++ b/test/sanity/compile/python2.7-skip.txt @@ -1,3 +1,3 @@ # The following are only run by release engineers who can be asked to have newer Python3 on their systems -hacking/release-announcement.py -hacking/porting-guide.py +hacking/build_library/build_ansible/command_plugins/porting_guide.py +hacking/build_library/build_ansible/command_plugins/release_announcement.py diff --git a/test/sanity/compile/python3.5-skip.txt b/test/sanity/compile/python3.5-skip.txt index 60a60b5c76..5814217d4a 100644 --- a/test/sanity/compile/python3.5-skip.txt +++ b/test/sanity/compile/python3.5-skip.txt @@ -1,3 +1,3 @@ # The following are only run by release engineers who can be asked to have newer Python3 on their systems -hacking/release-announcement.py -hacking/porting-guide.py +hacking/build_library/build_ansible/command_plugins/porting_guide.py +hacking/build_library/build_ansible/command_plugins/release_announcement.py