diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index a3764eaba5..1d6fd3ef4e 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, print_function -import glob import os import tempfile import time @@ -14,7 +13,6 @@ import random import pipes import string import atexit -import re import lib.pytar import lib.thread @@ -38,7 +36,6 @@ from lib.util import ( MissingEnvironmentVariable, display, run_command, - deepest_path, common_environment, remove_tree, make_dirs, @@ -58,7 +55,6 @@ from lib.target import ( walk_windows_integration_targets, walk_units_targets, walk_compile_targets, - walk_sanity_targets, ) from lib.changes import ( @@ -137,6 +133,10 @@ def install_command_requirements(args): if args.coverage: cmd += ['coverage'] + if isinstance(args, SanityConfig): + if args.junit: + cmd += ['junit-xml'] + try: run_command(args, cmd) except SubprocessError as ex: @@ -709,298 +709,6 @@ def command_compile(args): run_command(args, command) -def command_sanity(args): - """ - :type args: SanityConfig - """ - changes = get_changes_filter(args) - require = (args.require or []) + changes - targets = SanityTargets(args.include, args.exclude, require) - - if not targets.include: - raise AllTargetsSkipped() - - if args.delegate: - raise Delegate(require=changes) - - install_command_requirements(args) - - tests = SANITY_TESTS - - if args.test: - tests = [t for t in tests if t.name in args.test] - - if args.skip_test: - tests = [t for t in tests if t.name not in args.skip_test] - - for test in tests: - if args.list_tests: - display.info(test.name) - continue - - if test.intercept: - versions = SUPPORTED_PYTHON_VERSIONS - else: - versions = None, - - for version in versions: - if args.python and version and version != args.python: - continue - - display.info('Sanity check using %s%s' % (test.name, ' with Python %s' % version if version else '')) - - if test.intercept: - test.func(args, targets, python_version=version) - else: - test.func(args, targets) - - -def command_sanity_code_smell(args, _): - """ - :type args: SanityConfig - :type _: SanityTargets - """ - with open('test/sanity/code-smell/skip.txt', 'r') as skip_fd: - skip_tests = skip_fd.read().splitlines() - - tests = glob.glob('test/sanity/code-smell/*') - tests = sorted(p for p in tests - if os.access(p, os.X_OK) - and os.path.isfile(p) - and os.path.basename(p) not in skip_tests) - - env = ansible_environment(args) - - for test in tests: - display.info('Code smell check using %s' % os.path.basename(test)) - run_command(args, [test], env=env) - - -def command_sanity_validate_modules(args, targets): - """ - :type args: SanityConfig - :type targets: SanityTargets - """ - env = ansible_environment(args) - - paths = [deepest_path(i.path, 'lib/ansible/modules/') for i in targets.include_external] - paths = sorted(set(p for p in paths if p)) - - if not paths: - display.info('No tests applicable.', verbosity=1) - return - - cmd = ['test/sanity/validate-modules/validate-modules'] + paths - - with open('test/sanity/validate-modules/skip.txt', 'r') as skip_fd: - skip_paths = skip_fd.read().splitlines() - - skip_paths += [e.path for e in targets.exclude_external] - - if skip_paths: - cmd += ['--exclude', '^(%s)' % '|'.join(skip_paths)] - - if args.base_branch: - cmd.extend([ - '--base-branch', args.base_branch, - ]) - else: - display.warning('Cannot perform module comparison against the base branch. Base branch not detected when running locally.') - - run_command(args, cmd, env=env) - - -def command_sanity_shellcheck(args, targets): - """ - :type args: SanityConfig - :type targets: SanityTargets - """ - with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd: - skip_paths = set(skip_fd.read().splitlines()) - - with open('test/sanity/shellcheck/exclude.txt', 'r') as exclude_fd: - exclude = set(exclude_fd.read().splitlines()) - - paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths) - - if not paths: - display.info('No tests applicable.', verbosity=1) - return - - run_command(args, ['shellcheck', '-e', ','.join(sorted(exclude))] + paths) - - -def command_sanity_pep8(args, targets): - """ - :type args: SanityConfig - :type targets: SanityTargets - """ - skip_path = 'test/sanity/pep8/skip.txt' - legacy_path = 'test/sanity/pep8/legacy-files.txt' - - with open(skip_path, 'r') as skip_fd: - skip_paths = set(skip_fd.read().splitlines()) - - with open(legacy_path, 'r') as legacy_fd: - legacy_paths = set(legacy_fd.read().splitlines()) - - with open('test/sanity/pep8/legacy-ignore.txt', 'r') as ignore_fd: - legacy_ignore = set(ignore_fd.read().splitlines()) - - with open('test/sanity/pep8/current-ignore.txt', 'r') as ignore_fd: - current_ignore = sorted(ignore_fd.read().splitlines()) - - paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' and i.path not in skip_paths) - - if not paths: - display.info('No tests applicable.', verbosity=1) - return - - cmd = [ - 'pep8', - '--max-line-length', '160', - '--config', '/dev/null', - '--ignore', ','.join(sorted(current_ignore)), - ] + paths - - try: - stdout, stderr = run_command(args, cmd, capture=True) - status = 0 - except SubprocessError as ex: - stdout = ex.stdout - stderr = ex.stderr - status = ex.status - - if stderr: - raise SubprocessError(cmd=cmd, status=status, stderr=stderr) - - if args.explain: - return - - pattern = '^(?P[^:]*):(?P[0-9]+):(?P[0-9]+): (?P[A-Z0-9]{4}) (?P.*)$' - - results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()] - - for result in results: - for key in 'line', 'column': - result[key] = int(result[key]) - - failed_result_paths = set([result['path'] for result in results]) - passed_legacy_paths = set([path for path in paths if path in legacy_paths and path not in failed_result_paths]) - - errors = [] - summary = {} - - for path in sorted(passed_legacy_paths): - # Keep files out of the list which no longer require the relaxed rule set. - errors.append('PEP 8: %s: Passes current rule set. Remove from legacy list (%s).' % (path, legacy_path)) - - for path in sorted(skip_paths): - if not os.path.exists(path): - # Keep files out of the list which no longer exist in the repo. - errors.append('PEP 8: %s: Does not exist. Remove from skip list (%s).' % (path, skip_path)) - - for path in sorted(legacy_paths): - if not os.path.exists(path): - # Keep files out of the list which no longer exist in the repo. - errors.append('PEP 8: %s: Does not exist. Remove from legacy list (%s).' % (path, legacy_path)) - - for result in results: - path = result['path'] - line = result['line'] - column = result['column'] - code = result['code'] - message = result['message'] - - msg = 'PEP 8: %s:%s:%s: %s %s' % (path, line, column, code, message) - - if path in legacy_paths: - msg += ' (legacy)' - else: - msg += ' (current)' - - if path in legacy_paths and code in legacy_ignore: - # Files on the legacy list are permitted to have errors on the legacy ignore list. - # However, we want to report on their existence to track progress towards eliminating these exceptions. - display.info(msg, verbosity=3) - - key = '%s %s' % (code, re.sub('[0-9]+', 'NNN', message)) - - if key not in summary: - summary[key] = 0 - - summary[key] += 1 - else: - # Files not on the legacy list and errors not on the legacy ignore list are PEP 8 policy errors. - errors.append(msg) - - for error in errors: - display.error(error) - - if summary: - lines = [] - count = 0 - - for key in sorted(summary): - count += summary[key] - lines.append('PEP 8: %5d %s' % (summary[key], key)) - - display.info('PEP 8: There were %d different legacy issues found (%d total):' % - (len(summary), count), verbosity=1) - - display.info('PEP 8: Count Code Message', verbosity=1) - - for line in lines: - display.info(line, verbosity=1) - - if errors: - raise ApplicationError('PEP 8: There are %d issues which need to be resolved.' % len(errors)) - - -def command_sanity_yamllint(args, targets): - """ - :type args: SanityConfig - :type targets: SanityTargets - """ - paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml')) - - if not paths: - display.info('No tests applicable.', verbosity=1) - return - - run_command(args, ['yamllint'] + paths) - - -def command_sanity_ansible_doc(args, targets, python_version): - """ - :type args: SanityConfig - :type targets: SanityTargets - :type python_version: str - """ - with open('test/sanity/ansible-doc/skip.txt', 'r') as skip_fd: - skip_modules = set(skip_fd.read().splitlines()) - - modules = sorted(set(m for i in targets.include_external for m in i.modules) - - set(m for i in targets.exclude_external for m in i.modules) - - skip_modules) - - if not modules: - display.info('No tests applicable.', verbosity=1) - return - - env = ansible_environment(args) - cmd = ['ansible-doc'] + modules - - stdout, stderr = intercept_command(args, cmd, env=env, capture=True, python_version=python_version) - - if stderr: - display.error('Output on stderr from ansible-doc is considered an error.') - raise SubprocessError(cmd, stderr=stderr) - - if stdout: - display.info(stdout.strip(), verbosity=3) - - def intercept_command(args, cmd, capture=False, env=None, data=None, cwd=None, python_version=None): """ :type args: TestConfig @@ -1306,40 +1014,6 @@ class NoTestsForChanges(ApplicationWarning): super(NoTestsForChanges, self).__init__('No tests found for detected changes.') -class SanityTargets(object): - """Sanity test target information.""" - def __init__(self, include, exclude, require): - """ - :type include: list[str] - :type exclude: list[str] - :type require: list[str] - """ - self.all = not include - self.targets = tuple(sorted(walk_sanity_targets())) - self.include = walk_internal_targets(self.targets, include, exclude, require) - self.include_external, self.exclude_external = walk_external_targets(self.targets, include, exclude, require) - - -class SanityTest(object): - """Sanity test base class.""" - def __init__(self, name): - self.name = name - - -class SanityFunc(SanityTest): - """Sanity test function information.""" - def __init__(self, name, func, intercept=True): - """ - :type name: str - :type func: (SanityConfig, SanityTargets) -> None - :type intercept: bool - """ - super(SanityFunc, self).__init__(name) - - self.func = func - self.intercept = intercept - - class TestConfig(EnvironmentConfig): """Configuration common to all test commands.""" def __init__(self, args, command): @@ -1384,6 +1058,8 @@ class SanityConfig(TestConfig): self.test = args.test # type: list [str] self.skip_test = args.skip_test # type: list [str] self.list_tests = args.list_tests # type: bool + self.lint = args.lint # type: bool + self.junit = args.junit # type: bool if args.base_branch: self.base_branch = args.base_branch # str @@ -1485,15 +1161,3 @@ class AllTargetsSkipped(ApplicationWarning): """All targets skipped.""" def __init__(self): super(AllTargetsSkipped, self).__init__('All targets skipped.') - - -SANITY_TESTS = ( - # tests which ignore include/exclude (they're so fast it doesn't matter) - SanityFunc('code-smell', command_sanity_code_smell, intercept=False), - # tests which honor include/exclude - SanityFunc('shellcheck', command_sanity_shellcheck, intercept=False), - SanityFunc('pep8', command_sanity_pep8, intercept=False), - SanityFunc('yamllint', command_sanity_yamllint, intercept=False), - SanityFunc('validate-modules', command_sanity_validate_modules, intercept=False), - SanityFunc('ansible-doc', command_sanity_ansible_doc), -) diff --git a/test/runner/lib/sanity.py b/test/runner/lib/sanity.py new file mode 100644 index 0000000000..ad554f2013 --- /dev/null +++ b/test/runner/lib/sanity.py @@ -0,0 +1,827 @@ +"""Execute Ansible sanity tests.""" + +from __future__ import absolute_import, print_function + +import datetime +import glob +import json +import os +import re + +from xml.etree.ElementTree import ( + fromstring, + Element, +) + +from lib.util import ( + ApplicationError, + SubprocessError, + display, + run_command, + deepest_path, +) + +from lib.ansible_util import ( + ansible_environment, +) + +from lib.target import ( + walk_external_targets, + walk_internal_targets, + walk_sanity_targets, +) + +from lib.executor import ( + get_changes_filter, + AllTargetsSkipped, + Delegate, + install_command_requirements, + SUPPORTED_PYTHON_VERSIONS, + intercept_command, + SanityConfig, +) + +PEP8_SKIP_PATH = 'test/sanity/pep8/skip.txt' +PEP8_LEGACY_PATH = 'test/sanity/pep8/legacy-files.txt' + + +def command_sanity(args): + """ + :type args: SanityConfig + """ + changes = get_changes_filter(args) + require = (args.require or []) + changes + targets = SanityTargets(args.include, args.exclude, require) + + if not targets.include: + raise AllTargetsSkipped() + + if args.delegate: + raise Delegate(require=changes) + + install_command_requirements(args) + + tests = sanity_get_tests() + + if args.test: + tests = [t for t in tests if t.name in args.test] + + if args.skip_test: + tests = [t for t in tests if t.name not in args.skip_test] + + total = 0 + failed = [] + + for test in tests: + if args.list_tests: + display.info(test.name) + continue + + if test.intercept: + versions = SUPPORTED_PYTHON_VERSIONS + else: + versions = None, + + for version in versions: + if args.python and version and version != args.python: + continue + + display.info('Sanity check using %s%s' % (test.name, ' with Python %s' % version if version else '')) + + options = '' + + if test.script: + result = test.func(args, targets, test.script) + elif test.intercept: + result = test.func(args, targets, python_version=version) + options = ' --python %s' % version + else: + result = test.func(args, targets) + + result.write(args) + + total += 1 + + if isinstance(result, SanityFailure): + failed.append(result.test + options) + + if failed: + raise ApplicationError('The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % ( + len(failed), total, '\n'.join(failed))) + + +def command_sanity_code_smell(args, _, script): + """ + :type args: SanityConfig + :type _: SanityTargets + :type script: str + :rtype: SanityResult + """ + test = os.path.splitext(os.path.basename(script))[0] + + cmd = [script] + env = ansible_environment(args) + + # Since the output from scripts end up in other places besides the console, we don't want color here. + env.pop('ANSIBLE_FORCE_COLOR') + + try: + stdout, stderr = run_command(args, cmd, env=env, capture=True) + status = 0 + except SubprocessError as ex: + stdout = ex.stdout + stderr = ex.stderr + status = ex.status + + if stderr or status: + summary = str(SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)) + return SanityFailure(test, summary=summary) + + return SanitySuccess(test) + + +def command_sanity_validate_modules(args, targets): + """ + :type args: SanityConfig + :type targets: SanityTargets + :rtype: SanityResult + """ + test = 'validate-modules' + env = ansible_environment(args) + + paths = [deepest_path(i.path, 'lib/ansible/modules/') for i in targets.include_external] + paths = sorted(set(p for p in paths if p)) + + if not paths: + return SanitySkipped(test) + + cmd = [ + 'test/sanity/validate-modules/validate-modules', + '--format', 'json', + ] + paths + + with open('test/sanity/validate-modules/skip.txt', 'r') as skip_fd: + skip_paths = skip_fd.read().splitlines() + + skip_paths += [e.path for e in targets.exclude_external] + + if skip_paths: + cmd += ['--exclude', '^(%s)' % '|'.join(skip_paths)] + + if args.base_branch: + cmd.extend([ + '--base-branch', args.base_branch, + ]) + else: + display.warning('Cannot perform module comparison against the base branch. Base branch not detected when running locally.') + + try: + stdout, stderr = run_command(args, cmd, env=env, capture=True) + status = 0 + except SubprocessError as ex: + stdout = ex.stdout + stderr = ex.stderr + status = ex.status + + if stderr or status not in (0, 3): + raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) + + if args.explain: + return SanitySkipped(test) + + messages = json.loads(stdout) + + results = [] + + for filename in messages: + output = messages[filename] + + for item in output['errors']: + results.append(SanityMessage( + path=filename, + line=int(item['line']) if 'line' in item else 0, + column=int(item['column']) if 'column' in item else 0, + level='error', + code='E%s' % item['code'], + message=item['msg'], + )) + + if results: + return SanityFailure(test, messages=results) + + return SanitySuccess(test) + + +def command_sanity_shellcheck(args, targets): + """ + :type args: SanityConfig + :type targets: SanityTargets + :rtype: SanityResult + """ + test = 'shellcheck' + + with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd: + skip_paths = set(skip_fd.read().splitlines()) + + with open('test/sanity/shellcheck/exclude.txt', 'r') as exclude_fd: + exclude = set(exclude_fd.read().splitlines()) + + paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths) + + if not paths: + return SanitySkipped(test) + + cmd = [ + 'shellcheck', + '-e', ','.join(sorted(exclude)), + '--format', 'checkstyle', + ] + paths + + try: + stdout, stderr = run_command(args, cmd, capture=True) + status = 0 + except SubprocessError as ex: + stdout = ex.stdout + stderr = ex.stderr + status = ex.status + + if stderr or status > 1: + raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout) + + if args.explain: + return SanitySkipped(test) + + # json output is missing file paths in older versions of shellcheck, so we'll use xml instead + root = fromstring(stdout) # type: Element + + results = [] + + for item in root: # type: Element + for entry in item: # type: Element + results.append(SanityMessage( + message=entry.attrib['message'], + path=item.attrib['name'], + line=int(entry.attrib['line']), + column=int(entry.attrib['column']), + level=entry.attrib['severity'], + code=entry.attrib['source'].replace('ShellCheck.', ''), + )) + + if results: + return SanityFailure(test, messages=results) + + return SanitySuccess(test) + + +def command_sanity_pep8(args, targets): + """ + :type args: SanityConfig + :type targets: SanityTargets + :rtype: SanityResult + """ + test = 'pep8' + + with open(PEP8_SKIP_PATH, 'r') as skip_fd: + skip_paths = skip_fd.read().splitlines() + + with open(PEP8_LEGACY_PATH, 'r') as legacy_fd: + legacy_paths = legacy_fd.read().splitlines() + + with open('test/sanity/pep8/legacy-ignore.txt', 'r') as ignore_fd: + legacy_ignore = set(ignore_fd.read().splitlines()) + + with open('test/sanity/pep8/current-ignore.txt', 'r') as ignore_fd: + current_ignore = sorted(ignore_fd.read().splitlines()) + + skip_paths_set = set(skip_paths) + legacy_paths_set = set(legacy_paths) + + paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' and i.path not in skip_paths_set) + + if not paths: + return SanitySkipped(test) + + cmd = [ + 'pep8', + '--max-line-length', '160', + '--config', '/dev/null', + '--ignore', ','.join(sorted(current_ignore)), + ] + paths + + try: + stdout, stderr = run_command(args, cmd, capture=True) + status = 0 + except SubprocessError as ex: + stdout = ex.stdout + stderr = ex.stderr + status = ex.status + + if stderr: + raise SubprocessError(cmd=cmd, status=status, stderr=stderr) + + if args.explain: + return SanitySkipped(test) + + pattern = '^(?P[^:]*):(?P[0-9]+):(?P[0-9]+): (?P[WE][0-9]{3}) (?P.*)$' + + results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()] + + results = [SanityMessage( + message=r['message'], + path=r['path'], + line=int(r['line']), + column=int(r['column']), + level='warning' if r['code'].startswith('W') else 'error', + code=r['code'], + ) for r in results] + + failed_result_paths = set([result.path for result in results]) + used_paths = set(paths) + + errors = [] + summary = {} + + line = 0 + + for path in legacy_paths: + line += 1 + + if not os.path.exists(path): + # Keep files out of the list which no longer exist in the repo. + errors.append(SanityMessage( + code='A101', + message='Remove "%s" since it does not exist' % path, + path=PEP8_LEGACY_PATH, + line=line, + column=1, + )) + + if path in used_paths and path not in failed_result_paths: + # Keep files out of the list which no longer require the relaxed rule set. + errors.append(SanityMessage( + code='A201', + message='Remove "%s" since it passes the current rule set' % path, + path=PEP8_LEGACY_PATH, + line=line, + column=1, + )) + + line = 0 + + for path in skip_paths: + line += 1 + + if not os.path.exists(path): + # Keep files out of the list which no longer exist in the repo. + errors.append(SanityMessage( + code='A101', + message='Remove "%s" since it does not exist' % path, + path=PEP8_SKIP_PATH, + line=line, + column=1, + )) + + for result in results: + if result.path in legacy_paths_set and result.code in legacy_ignore: + # Files on the legacy list are permitted to have errors on the legacy ignore list. + # However, we want to report on their existence to track progress towards eliminating these exceptions. + display.info('PEP 8: %s (legacy)' % result, verbosity=3) + + key = '%s %s' % (result.code, re.sub('[0-9]+', 'NNN', result.message)) + + if key not in summary: + summary[key] = 0 + + summary[key] += 1 + else: + # Files not on the legacy list and errors not on the legacy ignore list are PEP 8 policy errors. + errors.append(result) + + if summary: + lines = [] + count = 0 + + for key in sorted(summary): + count += summary[key] + lines.append('PEP 8: %5d %s' % (summary[key], key)) + + display.info('PEP 8: There were %d different legacy issues found (%d total):' % (len(summary), count), verbosity=1) + display.info('PEP 8: Count Code Message', verbosity=1) + + for line in lines: + display.info(line, verbosity=1) + + if errors: + return SanityFailure(test, messages=errors) + + return SanitySuccess(test) + + +def command_sanity_yamllint(args, targets): + """ + :type args: SanityConfig + :type targets: SanityTargets + :rtype: SanityResult + """ + test = 'yamllint' + + paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml')) + + if not paths: + return SanitySkipped(test) + + cmd = [ + 'yamllint', + '--format', 'parsable', + ] + paths + + try: + stdout, stderr = run_command(args, cmd, capture=True) + status = 0 + except SubprocessError as ex: + stdout = ex.stdout + stderr = ex.stderr + status = ex.status + + if stderr: + raise SubprocessError(cmd=cmd, status=status, stderr=stderr) + + if args.explain: + return SanitySkipped(test) + + pattern = r'^(?P[^:]*):(?P[0-9]+):(?P[0-9]+): \[(?Pwarning|error)\] (?P.*)$' + + results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()] + + results = [SanityMessage( + message=r['message'], + path=r['path'], + line=int(r['line']), + column=int(r['column']), + level=r['level'], + ) for r in results] + + if results: + return SanityFailure(test, messages=results) + + return SanitySuccess(test) + + +def command_sanity_ansible_doc(args, targets, python_version): + """ + :type args: SanityConfig + :type targets: SanityTargets + :type python_version: str + :rtype: SanityResult + """ + test = 'ansible-doc' + + with open('test/sanity/ansible-doc/skip.txt', 'r') as skip_fd: + skip_modules = set(skip_fd.read().splitlines()) + + modules = sorted(set(m for i in targets.include_external for m in i.modules) - + set(m for i in targets.exclude_external for m in i.modules) - + skip_modules) + + if not modules: + return SanitySkipped(test, python_version=python_version) + + env = ansible_environment(args) + cmd = ['ansible-doc'] + modules + + try: + stdout, stderr = intercept_command(args, cmd, env=env, capture=True, python_version=python_version) + status = 0 + except SubprocessError as ex: + stdout = ex.stdout + stderr = ex.stderr + status = ex.status + + if status: + summary = str(SubprocessError(cmd=cmd, status=status, stderr=stderr)) + return SanityFailure(test, summary=summary, python_version=python_version) + + if stdout: + display.info(stdout.strip(), verbosity=3) + + if stderr: + summary = 'Output on stderr from ansible-doc is considered an error.\n\n%s' % SubprocessError(cmd, stderr=stderr) + return SanityFailure(test, summary=summary, python_version=python_version) + + return SanitySuccess(test, python_version=python_version) + + +def collect_code_smell_tests(): + """ + :rtype: tuple(SanityFunc) + """ + with open('test/sanity/code-smell/skip.txt', 'r') as skip_fd: + skip_tests = skip_fd.read().splitlines() + + paths = glob.glob('test/sanity/code-smell/*') + paths = sorted(p for p in paths + if os.access(p, os.X_OK) + and os.path.isfile(p) + and os.path.basename(p) not in skip_tests) + + tests = tuple(SanityFunc(os.path.splitext(os.path.basename(p))[0], command_sanity_code_smell, script=p, intercept=False) for p in paths) + + return tests + + +def sanity_init(): + """Initialize full sanity test list (includes code-smell scripts determined at runtime).""" + global SANITY_TESTS # pylint: disable=locally-disabled, global-statement + SANITY_TESTS = tuple(sorted(SANITY_TESTS + collect_code_smell_tests(), key=lambda k: k.name)) + + +def sanity_get_tests(): + """ + :rtype: tuple(SanityFunc) + """ + return SANITY_TESTS + + +class SanityResult(object): + """Base class for sanity test results.""" + def __init__(self, test, python_version=None): + """ + :type test: str + :type python_version: str + """ + self.test = test + self.python_version = python_version + + try: + import junit_xml + except ImportError: + junit_xml = None + + self.junit = junit_xml + + def write(self, args): + """ + :type args: SanityConfig + """ + self.write_console() + + if args.lint: + self.write_lint() + + if args.junit: + if self.junit: + self.write_junit(args) + else: + display.warning('Skipping junit xml output because the `junit-xml` python package was not found.', unique=True) + + def write_console(self): + """Write results to console.""" + pass + + def write_lint(self): + """Write lint results to stdout.""" + pass + + def write_junit(self, args): + """ + :type args: SanityConfig + """ + pass + + def save_junit(self, args, test_case, properties=None): + """ + :type args: SanityConfig + :type test_case: junit_xml.TestCase + :type properties: dict[str, str] | None + :rtype: str | None + """ + path = 'test/results/junit/ansible-test-%s' % self.test + + if self.python_version: + path += '-python-%s' % self.python_version + + path += '.xml' + + test_suites = [ + self.junit.TestSuite( + name='ansible-test', + test_cases=[test_case], + timestamp=datetime.datetime.utcnow().replace(microsecond=0).isoformat(), + properties=properties, + ), + ] + + report = self.junit.TestSuite.to_xml_string(test_suites=test_suites, prettyprint=True, encoding='utf-8') + + if args.explain: + return + + with open(path, 'wb') as xml: + xml.write(report.encode('utf-8', 'strict')) + + +class SanitySuccess(SanityResult): + """Sanity test success.""" + def __init__(self, test, python_version=None): + """ + :type test: str + :type python_version: str + """ + super(SanitySuccess, self).__init__(test, python_version) + + def write_junit(self, args): + """ + :type args: SanityConfig + """ + test_case = self.junit.TestCase(name=self.test) + + self.save_junit(args, test_case) + + +class SanitySkipped(SanityResult): + """Sanity test skipped.""" + def __init__(self, test, python_version=None): + """ + :type test: str + :type python_version: str + """ + super(SanitySkipped, self).__init__(test, python_version) + + def write_console(self): + """Write results to console.""" + display.info('No tests applicable.', verbosity=1) + + def write_junit(self, args): + """ + :type args: SanityConfig + """ + test_case = self.junit.TestCase(name=self.test) + test_case.add_skipped_info('No tests applicable.') + + self.save_junit(args, test_case) + + +class SanityFailure(SanityResult): + """Sanity test failure.""" + def __init__(self, test, python_version=None, messages=None, summary=None): + """ + :type test: str + :type python_version: str + :type messages: list[SanityMessage] + :type summary: str + """ + super(SanityFailure, self).__init__(test, python_version) + + self.messages = messages + self.summary = summary + + def write_console(self): + """Write results to console.""" + if self.summary: + display.error(self.summary) + else: + display.error('Found %d %s issue(s) which need to be resolved:' % (len(self.messages), self.test)) + + for message in self.messages: + display.error(message) + + def write_lint(self): + """Write lint results to stdout.""" + if self.summary: + command = self.format_command() + message = 'The test `%s` failed. See stderr output for details.' % command + path = 'test/runner/ansible-test' + message = SanityMessage(message, path) + print(message) + else: + for message in self.messages: + print(message) + + def write_junit(self, args): + """ + :type args: SanityConfig + """ + title = self.format_title() + output = self.format_block() + + # Hack to remove ANSI color reset code from SubprocessError messages. + output = output.replace(display.clear, '') + + test_case = self.junit.TestCase(classname='sanity', name=self.test) + + # Include a leading newline to improve readability on Shippable "Tests" tab. + # Without this, the first line becomes indented. + test_case.add_failure_info(message=title, output='\n%s' % output) + + self.save_junit(args, test_case) + + def format_command(self): + """ + :rtype: str + """ + command = 'ansible-test sanity --test %s' % self.test + + if self.python_version: + command += ' --python %s' % self.python_version + + return command + + def format_title(self): + """ + :rtype: str + """ + command = self.format_command() + + if self.summary: + reason = 'error' + else: + reason = 'error' if len(self.messages) == 1 else 'errors' + + title = 'The test `%s` failed with the following %s:' % (command, reason) + + return title + + def format_block(self): + """ + :rtype: str + """ + if self.summary: + block = self.summary + else: + block = '\n'.join(str(m) for m in self.messages) + + message = block.strip() + + return message + + +class SanityMessage(object): + """Single sanity test message for one file.""" + def __init__(self, message, path, line=0, column=0, level='error', code=None): + """ + :type message: str + :type path: str + :type line: int + :type column: int + :type level: str + :type code: str | None + """ + self.path = path + self.line = line + self.column = column + self.level = level + self.code = code + self.message = message + + def __str__(self): + if self.code: + msg = '%s %s' % (self.code, self.message) + else: + msg = self.message + + return '%s:%s:%s: %s' % (self.path, self.line, self.column, msg) + + +class SanityTargets(object): + """Sanity test target information.""" + def __init__(self, include, exclude, require): + """ + :type include: list[str] + :type exclude: list[str] + :type require: list[str] + """ + self.all = not include + self.targets = tuple(sorted(walk_sanity_targets())) + self.include = walk_internal_targets(self.targets, include, exclude, require) + self.include_external, self.exclude_external = walk_external_targets(self.targets, include, exclude, require) + + +class SanityTest(object): + """Sanity test base class.""" + def __init__(self, name): + self.name = name + + +class SanityFunc(SanityTest): + """Sanity test function information.""" + def __init__(self, name, func, intercept=True, script=None): + """ + :type name: str + :type func: (SanityConfig, SanityTargets) -> SanityResult + :type intercept: bool + :type script: str | None + """ + super(SanityFunc, self).__init__(name) + + self.func = func + self.intercept = intercept + self.script = script + + +SANITY_TESTS = ( + SanityFunc('shellcheck', command_sanity_shellcheck, intercept=False), + SanityFunc('pep8', command_sanity_pep8, intercept=False), + SanityFunc('yamllint', command_sanity_yamllint, intercept=False), + SanityFunc('validate-modules', command_sanity_validate_modules, intercept=False), + SanityFunc('ansible-doc', command_sanity_ansible_doc), +) diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index f500ba480e..1b9084f657 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -287,6 +287,8 @@ class Display(object): self.verbosity = 0 self.color = True self.warnings = [] + self.warnings_unique = set() + self.info_stderr = False def __warning(self, message): """ @@ -304,10 +306,17 @@ class Display(object): for warning in self.warnings: self.__warning(warning) - def warning(self, message): + def warning(self, message, unique=False): """ :type message: str + :type unique: bool """ + if unique: + if message in self.warnings_unique: + return + + self.warnings_unique.add(message) + self.__warning(message) self.warnings.append(message) @@ -330,7 +339,7 @@ class Display(object): """ if self.verbosity >= verbosity: color = self.verbosity_colors.get(verbosity, self.yellow) - self.print_message(message, color=color) + self.print_message(message, color=color, fd=sys.stderr if self.info_stderr else sys.stdout) def print_message(self, message, color=None, fd=sys.stdout): # pylint: disable=locally-disabled, invalid-name """ diff --git a/test/runner/test.py b/test/runner/test.py index c6a5ed613a..4b7a8b6227 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -24,9 +24,7 @@ from lib.executor import ( command_windows_integration, command_units, command_compile, - command_sanity, command_shell, - SANITY_TESTS, SUPPORTED_PYTHON_VERSIONS, COMPILE_PYTHON_VERSIONS, PosixIntegrationConfig, @@ -42,6 +40,12 @@ from lib.executor import ( check_startup, ) +from lib.sanity import ( + command_sanity, + sanity_init, + sanity_get_tests, +) + from lib.target import ( find_target_completion, walk_posix_integration_targets, @@ -64,10 +68,12 @@ def main(): try: git_root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')) os.chdir(git_root) + sanity_init() args = parse_args() config = args.config(args) display.verbosity = config.verbosity display.color = config.color + display.info_stderr = isinstance(config, SanityConfig) and config.lint check_startup() try: @@ -267,19 +273,27 @@ def parse_args(): sanity.add_argument('--test', metavar='TEST', action='append', - choices=[t.name for t in SANITY_TESTS], - help='tests to run') + choices=[test.name for test in sanity_get_tests()], + help='tests to run').completer = complete_sanity_test sanity.add_argument('--skip-test', metavar='TEST', action='append', - choices=[t.name for t in SANITY_TESTS], - help='tests to skip') + choices=[test.name for test in sanity_get_tests()], + help='tests to skip').completer = complete_sanity_test sanity.add_argument('--list-tests', action='store_true', help='list available tests') + sanity.add_argument('--lint', + action='store_true', + help='write lint output to stdout, everything else stderr') + + sanity.add_argument('--junit', + action='store_true', + help='write test failures to junit xml files') + sanity.add_argument('--python', metavar='VERSION', choices=SUPPORTED_PYTHON_VERSIONS, @@ -540,5 +554,18 @@ def complete_network_platform(prefix, parsed_args, **_): return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)] +def complete_sanity_test(prefix, parsed_args, **_): + """ + :type prefix: unicode + :type parsed_args: any + :rtype: list[str] + """ + del parsed_args + + tests = sorted(t.name for t in sanity_get_tests()) + + return [i for i in tests if i.startswith(prefix)] + + if __name__ == '__main__': main() diff --git a/test/sanity/code-smell/no-iterkeys.sh b/test/sanity/code-smell/no-iterkeys.sh index bc21b966b0..7323309ec8 100755 --- a/test/sanity/code-smell/no-iterkeys.sh +++ b/test/sanity/code-smell/no-iterkeys.sh @@ -3,7 +3,9 @@ ITERKEYS_USERS=$(grep -r -I iterkeys . \ --exclude-dir .git \ --exclude-dir .tox \ + --exclude-dir .idea \ --exclude-dir docsite \ + --exclude-dir results \ | grep -v \ -e lib/ansible/compat/six/_six.py \ -e lib/ansible/module_utils/six.py \ diff --git a/test/utils/shippable/download-logs b/test/utils/shippable/download-logs deleted file mode 100755 index 48143f8846..0000000000 --- a/test/utils/shippable/download-logs +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -# (c) 2016 Red Hat, Inc. -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import print_function - -import os -import requests - -from argparse import ArgumentParser - - -def main(): - api_key = get_api_key() - - parser = ArgumentParser(description='Download logs from all jobs in a Shippable run.') - - parser.add_argument('run_id', - help='shippable run id.') - - parser.add_argument('-v', '--verbose', - dest='verbose', - action='store_true', - help='show what is being downloaded') - - parser.add_argument('-t', '--test', - dest='test', - action='store_true', - help='show what would be downloaded without downloading') - - parser.add_argument('--key', - dest='api_key', - default=api_key, - required=api_key is None, - help='api key for accessing Shippable') - - args = parser.parse_args() - - headers = dict( - Authorization='apiToken %s' % args.api_key, - ) - - response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers) - - if response.status_code != 200: - raise Exception(response.content) - - body = response.json() - output_dir = args.run_id - - if not args.test: - if not os.path.exists(output_dir): - os.mkdir(output_dir) - - for j in body: - job_id = j['id'] - job_number = j['jobNumber'] - path = os.path.join(output_dir, '%s.log' % job_number) - url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id - - if args.verbose or args.test: - print('%s' % path) - - if os.path.exists(path): - continue - - if not args.test: - response = requests.get(url, headers=headers) - - if response.status_code != 200: - raise Exception(response.content) - - log = response.content - - with open(path, 'w') as f: - f.write(log) - - -def get_api_key(): - path = os.path.join(os.environ['HOME'], '.shippable.key') - - try: - with open(path, 'r') as f: - return f.read().strip() - except IOError: - return None - - -if __name__ == '__main__': - main() diff --git a/test/utils/shippable/download.py b/test/utils/shippable/download.py new file mode 100755 index 0000000000..cd5609df2f --- /dev/null +++ b/test/utils/shippable/download.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK + +# (c) 2016 Red Hat, Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from __future__ import print_function + +import json +import os +import re +import requests + +from argparse import ArgumentParser + +try: + import argcomplete +except ImportError: + argcomplete = None + + +def main(): + api_key = get_api_key() + + parser = ArgumentParser(description='Download results from a Shippable run.') + + parser.add_argument('run_id', + help='shippable run id.') + + parser.add_argument('-v', '--verbose', + dest='verbose', + action='store_true', + help='show what is being downloaded') + + parser.add_argument('-t', '--test', + dest='test', + action='store_true', + help='show what would be downloaded without downloading') + + parser.add_argument('--key', + dest='api_key', + default=api_key, + required=api_key is None, + help='api key for accessing Shippable') + + parser.add_argument('--console-logs', + action='store_true', + help='download console logs') + + parser.add_argument('--test-results', + action='store_true', + help='download test results') + + parser.add_argument('--coverage-results', + action='store_true', + help='download code coverage results') + + parser.add_argument('--all', + action='store_true', + help='download everything') + + parser.add_argument('--job-number', + action='append', + type=int, + help='limit downloads to the given job number') + + if argcomplete: + argcomplete.autocomplete(parser) + + args = parser.parse_args() + + if args.all: + args.console_logs = True + args.test_results = True + args.coverage_results = True + + if not args.console_logs and not args.test_results and not args.coverage_results: + parser.error('At least one download option is required: --console-logs, --test-results, --coverage-results') + + headers = dict( + Authorization='apiToken %s' % args.api_key, + ) + + response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + body = response.json() + output_dir = args.run_id + + if not args.test: + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + for j in body: + job_id = j['id'] + job_number = j['jobNumber'] + + if args.job_number and job_number not in args.job_number: + continue + + if args.console_logs: + path = os.path.join(output_dir, '%s-console.log' % job_number) + url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id + download(args, headers, path, url) + + if args.test_results: + path = os.path.join(output_dir, '%s-test.json' % job_number) + url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id + download(args, headers, path, url) + extract_contents(args, path, os.path.join(output_dir, '%s-test' % job_number)) + + if args.coverage_results: + path = os.path.join(output_dir, '%s-coverage.json' % job_number) + url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id + download(args, headers, path, url) + extract_contents(args, path, os.path.join(output_dir, '%s-coverage' % job_number)) + + +def extract_contents(args, path, output_dir): + if not args.test: + with open(path, 'r') as json_fd: + items = json.load(json_fd) + + for item in items: + contents = item['contents'] + path = output_dir + '/' + re.sub('^/*', '', item['path']) + + directory = os.path.dirname(path) + + if not os.path.exists(directory): + os.makedirs(directory) + + if args.verbose: + print(path) + + if not os.path.exists(path): + with open(path, 'w') as output_fd: + output_fd.write(contents) + + +def download(args, headers, path, url): + if args.verbose or args.test: + print(path) + + if os.path.exists(path): + return + + if not args.test: + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + content = response.content + + with open(path, 'w') as f: + f.write(content) + + +def get_api_key(): + path = os.path.join(os.environ['HOME'], '.shippable.key') + + try: + with open(path, 'r') as f: + return f.read().strip() + except IOError: + return None + + +if __name__ == '__main__': + main() diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh index abdbd4f73f..a74a93900a 100755 --- a/test/utils/shippable/other.sh +++ b/test/utils/shippable/other.sh @@ -18,5 +18,5 @@ ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc retry.py pip install tox --disable-pip-version-check ansible-test compile --color -v -ansible-test sanity --color -v --tox --skip-test ansible-doc --python 2.7 -ansible-test sanity --color -v --tox --test ansible-doc --coverage +ansible-test sanity --color -v --junit --tox --skip-test ansible-doc --python 2.7 +ansible-test sanity --color -v --junit --tox --test ansible-doc --coverage diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh index 811f441260..084175eaa7 100755 --- a/test/utils/shippable/shippable.sh +++ b/test/utils/shippable/shippable.sh @@ -30,11 +30,11 @@ function cleanup { if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then ansible-test coverage xml --color -v --requirements - cp -av test/results/reports/coverage.xml shippable/codecoverage/coverage.xml + cp -a test/results/reports/coverage.xml shippable/codecoverage/coverage.xml fi rmdir shippable/testresults/ - cp -av test/results/junit/ shippable/testresults/ + cp -a test/results/junit/ shippable/testresults/ } trap cleanup EXIT