diff --git a/test/runner/lib/sanity/validate_modules.py b/test/runner/lib/sanity/validate_modules.py index 8b9da45d63..e52fd9990c 100644 --- a/test/runner/lib/sanity/validate_modules.py +++ b/test/runner/lib/sanity/validate_modules.py @@ -1,7 +1,9 @@ """Sanity test using validate-modules.""" from __future__ import absolute_import, print_function +import collections import json +import os from lib.sanity import ( SanitySingleVersion, @@ -26,6 +28,14 @@ from lib.config import ( SanityConfig, ) +from lib.test import ( + calculate_confidence, + calculate_best_confidence, +) + +VALIDATE_SKIP_PATH = 'test/sanity/validate-modules/skip.txt' +VALIDATE_IGNORE_PATH = 'test/sanity/validate-modules/ignore.txt' + class ValidateModulesTest(SanitySingleVersion): """Sanity test using validate-modules.""" @@ -49,9 +59,27 @@ class ValidateModulesTest(SanitySingleVersion): '--format', 'json', ] + paths - with open('test/sanity/validate-modules/skip.txt', 'r') as skip_fd: + with open(VALIDATE_SKIP_PATH, 'r') as skip_fd: skip_paths = skip_fd.read().splitlines() + invalid_ignores = [] + + with open(VALIDATE_IGNORE_PATH, 'r') as ignore_fd: + ignore_entries = ignore_fd.read().splitlines() + ignore = collections.defaultdict(dict) + line = 0 + + for ignore_entry in ignore_entries: + line += 1 + + if ' ' not in ignore_entry: + invalid_ignores.append((line, 'Invalid syntax')) + continue + + path, code = ignore_entry.split(' ', 1) + + ignore[path][code] = line + skip_paths += [e.path for e in targets.exclude_external] if skip_paths: @@ -80,13 +108,13 @@ class ValidateModulesTest(SanitySingleVersion): messages = json.loads(stdout) - results = [] + errors = [] for filename in messages: output = messages[filename] for item in output['errors']: - results.append(SanityMessage( + errors.append(SanityMessage( path=filename, line=int(item['line']) if 'line' in item else 0, column=int(item['column']) if 'column' in item else 0, @@ -95,7 +123,60 @@ class ValidateModulesTest(SanitySingleVersion): message=item['msg'], )) - if results: - return SanityFailure(self.name, messages=results) + filtered = [] + + for error in errors: + if error.code in ignore[error.path]: + ignore[error.path][error.code] = None # error ignored, clear line number of ignore entry to track usage + else: + filtered.append(error) # error not ignored + + errors = filtered + + for invalid_ignore in invalid_ignores: + errors.append(SanityMessage( + code='A201', + message=invalid_ignore[1], + path=VALIDATE_IGNORE_PATH, + line=invalid_ignore[0], + column=1, + confidence=calculate_confidence(VALIDATE_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None, + )) + + for path in skip_paths: + line += 1 + + if not os.path.exists(path): + # Keep files out of the list which no longer exist in the repo. + errors.append(SanityMessage( + code='A101', + message='Remove "%s" since it does not exist' % path, + path=VALIDATE_SKIP_PATH, + line=line, + column=1, + confidence=calculate_best_confidence(((VALIDATE_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, + )) + + for path in paths: + if path not in ignore: + continue + + for code in ignore[path]: + line = ignore[path][code] + + if not line: + continue + + errors.append(SanityMessage( + code='A102', + message='Remove since "%s" passes "%s" test' % (path, code), + path=VALIDATE_IGNORE_PATH, + line=line, + column=1, + confidence=calculate_best_confidence(((VALIDATE_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None, + )) + + if errors: + return SanityFailure(self.name, messages=errors) return SanitySuccess(self.name) diff --git a/test/sanity/validate-modules/ignore.txt b/test/sanity/validate-modules/ignore.txt new file mode 100644 index 0000000000..e69de29bb2