From dc3e16113d09e32fb750632b79ef224fccc8feaf Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 25 Jul 2021 10:00:10 +0200 Subject: [PATCH] Add BOTMETA extra sanity test (#3064) * Add BOTMETA sanity test. * Make compile with Python 2.6. --- .github/BOTMETA.yml | 7 +- tests/sanity/extra/botmeta.json | 11 ++ tests/sanity/extra/botmeta.py | 184 ++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 tests/sanity/extra/botmeta.json create mode 100755 tests/sanity/extra/botmeta.py diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 8df7297720..55f34d3041 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -159,11 +159,13 @@ files: $module_utils/redfish_utils.py: maintainers: $team_redfish labels: redfish_utils - $module_utils/remote_management/lxca/common.py: navalkp prabhosa + $module_utils/remote_management/lxca/common.py: + maintainers: navalkp prabhosa $module_utils/scaleway.py: maintainers: $team_scaleway labels: cloud scaleway - $module_utils/storage/hpe3par/hpe3par.py: farhan7500 gautamphegde + $module_utils/storage/hpe3par/hpe3par.py: + maintainers: farhan7500 gautamphegde $module_utils/utm_utils.py: maintainers: $team_e_spirit labels: utm_utils @@ -508,7 +510,6 @@ files: $modules/notification/osx_say.py: maintainers: ansible mpdehaan labels: _osx_say - deprecated: true $modules/notification/bearychat.py: maintainers: tonyseek $modules/notification/campfire.py: diff --git a/tests/sanity/extra/botmeta.json b/tests/sanity/extra/botmeta.json new file mode 100644 index 0000000000..cba49c90cd --- /dev/null +++ b/tests/sanity/extra/botmeta.json @@ -0,0 +1,11 @@ +{ + "include_symlinks": false, + "prefixes": [ + ".github/BOTMETA.yml" + ], + "output": "path-line-column-message", + "requirements": [ + "PyYAML", + "voluptuous==0.12.1" + ] +} diff --git a/tests/sanity/extra/botmeta.py b/tests/sanity/extra/botmeta.py new file mode 100755 index 0000000000..e8ea819394 --- /dev/null +++ b/tests/sanity/extra/botmeta.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +"""Check BOTMETA file.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import ast +import os +import re +import sys + +import yaml + +from voluptuous import All, Any, MultipleInvalid, PREVENT_EXTRA +from voluptuous import Required, Schema, Invalid +from voluptuous.humanize import humanize_error + + +REPORT_MISSING_MAINTAINERS = False + +FILENAME = '.github/BOTMETA.yml' + +LIST_ENTRIES = frozenset(('supershipit', 'maintainers', 'labels', 'keywords', 'notify', 'ignore')) + +AUTHOR_REGEX = re.compile(r'^\w.*\(@([\w-]+)\)(?![\w.])$') + + +def read_authors(filename): + data = {} + try: + with open(filename, 'rb') as b_module_data: + M = ast.parse(b_module_data.read()) + + for child in M.body: + if isinstance(child, ast.Assign): + for t in child.targets: + try: + theid = t.id + except AttributeError: + # skip errors can happen when trying to use the normal code + continue + + if theid == 'DOCUMENTATION': + if isinstance(child.value, ast.Dict): + data = ast.literal_eval(child.value) + else: + data = yaml.safe_load(child.value.s) + + except Exception as e: + print('%s:%d:%d: Cannot load DOCUMENTATION: %s' % (filename, 0, 0, e)) + return [] + + author = data.get('author') or [] + if isinstance(author, str): + author = [author] + return author + + +def validate(filename, filedata): + if filename.startswith('plugins/doc_fragments/'): + return + # Compile lis tof all active and inactive maintainers + all_maintainers = filedata['maintainers'] + filedata['ignore'] + if not all_maintainers: + if REPORT_MISSING_MAINTAINERS: + print('%s:%d:%d: %s' % (FILENAME, 0, 0, 'No (active or inactive) maintainer mentioned for %s' % filename)) + return + if filename.startswith('plugins/filter/'): + return + maintainers = read_authors(filename) + for maintainer in maintainers: + m = AUTHOR_REGEX.match(maintainer) + if m: + maintainer = m.group(1) + if maintainer not in all_maintainers: + msg = 'Author %s not mentioned as active or inactive maintainer for %s (mentioned are: %s)' % ( + maintainer, filename, ', '.join(all_maintainers)) + if REPORT_MISSING_MAINTAINERS: + print('%s:%d:%d: %s' % (FILENAME, 0, 0, msg)) + + +def main(): + """Main entry point.""" + paths = sys.argv[1:] or sys.stdin.read().splitlines() + paths = [path for path in paths if path.endswith('/aliases')] + + try: + with open(FILENAME, 'rb') as f: + botmeta = yaml.safe_load(f) + except yaml.error.MarkedYAMLError as ex: + print('%s:%d:%d: YAML load failed: %s' % (FILENAME, ex.context_mark.line + + 1, ex.context_mark.column + 1, re.sub(r'\s+', ' ', str(ex)))) + return + except Exception as ex: # pylint: disable=broad-except + print('%s:%d:%d: YAML load failed: %s' % + (FILENAME, 0, 0, re.sub(r'\s+', ' ', str(ex)))) + return + + # Validate schema + + MacroSchema = Schema({ + (str): str, + }, extra=PREVENT_EXTRA) + + FilesSchema = Schema({ + (str): { + ('supershipit'): str, + ('support'): Any('community'), + ('maintainers'): str, + ('labels'): str, + ('keywords'): str, + ('notify'): str, + ('ignore'): str, + }, + }, extra=PREVENT_EXTRA) + + schema = Schema({ + ('automerge'): bool, + ('macros'): MacroSchema, + ('files'): FilesSchema, + }, extra=PREVENT_EXTRA) + + try: + schema(botmeta) + except MultipleInvalid as ex: + for error in ex.errors: + # No way to get line/column numbers + print('%s:%d:%d: %s' % (FILENAME, 0, 0, humanize_error(botmeta, error))) + return + + # Preprocess (substitute macros, convert to lists) + macros = botmeta.get('macros') or {} + macro_re = re.compile(r'\$([a-zA-Z_]+)') + + def convert_macros(text, macros): + def f(m): + return macros[m.group(1)] + + return macro_re.sub(f, text) + + files = {} + try: + for file, filedata in (botmeta.get('files') or {}).items(): + file = convert_macros(file, macros) + filedata = dict((k, convert_macros(v, macros)) for k, v in filedata.items()) + files[file] = filedata + for k, v in filedata.items(): + if k in LIST_ENTRIES: + filedata[k] = v.split() + except KeyError as e: + print('%s:%d:%d: %s' % (FILENAME, 0, 0, 'Found unknown macro %s' % e)) + return + + # Scan all files + for dirpath, dirnames, filenames in os.walk('plugins/'): + for file in filenames: + if file.endswith('.pyc'): + continue + filename = os.path.join(dirpath, file) + if os.path.islink(filename): + continue + if os.path.isfile(filename): + matching_files = [] + for file, filedata in files.items(): + if filename.startswith(file): + matching_files.append((file, filedata)) + if not matching_files: + print('%s:%d:%d: %s' % (FILENAME, 0, 0, 'Did not find any entry for %s' % filename)) + + matching_files.sort(key=lambda kv: kv[0]) + filedata = dict() + for k in LIST_ENTRIES: + filedata[k] = [] + for dummy, data in matching_files: + for k, v in data.items(): + if k in LIST_ENTRIES: + v = filedata[k] + v + filedata[k] = v + validate(filename, filedata) + + +if __name__ == '__main__': + main()