diff --git a/test/runner/retry.py b/test/runner/retry.py new file mode 100755 index 0000000000..49721316f0 --- /dev/null +++ b/test/runner/retry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK +"""Automatically retry failed commands.""" + +from __future__ import absolute_import, print_function + +import argparse +import errno +import os +import sys +import time + +from lib.util import ( + display, + raw_command, + ApplicationError, + ApplicationWarning, + SubprocessError, +) + +try: + import argcomplete +except ImportError: + argcomplete = None + + +def main(): + """Main program function.""" + try: + args = parse_args() + display.verbosity = args.verbosity + display.color = args.color + + command = [args.command] + args.args + + for attempt in range(0, args.tries): + if attempt > 0: + time.sleep(args.sleep) + + try: + raw_command(command, env=os.environ) + return + except SubprocessError as ex: + display.error(ex) + except ApplicationWarning as ex: + display.warning(str(ex)) + exit(0) + except ApplicationError as ex: + display.error(str(ex)) + exit(1) + except KeyboardInterrupt: + exit(2) + except IOError as ex: + if ex.errno == errno.EPIPE: + exit(3) + raise + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser() + + parser.add_argument('-v', '--verbose', + dest='verbosity', + action='count', + default=0, + help='display more output') + + parser.add_argument('--color', + metavar='COLOR', + nargs='?', + help='generate color output: %(choices)s', + choices=('yes', 'no', 'auto'), + const='yes', + default='auto') + + parser.add_argument('--tries', + metavar='TRIES', + type=int, + default=3, + help='number of tries to execute command (default: %(default)s)') + + parser.add_argument('--sleep', + metavar='SECONDS', + type=int, + default=3, + help='seconds to sleep between tries (default: %(default)s)') + + parser.add_argument('command', + help='command to execute') + + parser.add_argument('args', + metavar='...', + nargs=argparse.REMAINDER, + help='optional arguments for command') + + if argcomplete: + argcomplete.autocomplete(parser) + + args = parser.parse_args() + + if args.color == 'yes': + args.color = True + elif args.color == 'no': + args.color = False + elif 'SHIPPABLE' in os.environ: + args.color = True + else: + args.color = sys.stdout.isatty() + + return args + + +if __name__ == '__main__': + main() diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh index 31ea17c2e9..abdbd4f73f 100755 --- a/test/utils/shippable/other.sh +++ b/test/utils/shippable/other.sh @@ -2,12 +2,12 @@ set -o pipefail -add-apt-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe' -add-apt-repository 'ppa:ubuntu-toolchain-r/test' -add-apt-repository 'ppa:fkrull/deadsnakes' +retry.py add-apt-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe' +retry.py add-apt-repository 'ppa:ubuntu-toolchain-r/test' +retry.py add-apt-repository 'ppa:fkrull/deadsnakes' -apt-get update -qq -apt-get install -qq \ +retry.py apt-get update -qq +retry.py apt-get install -qq \ shellcheck \ python2.4 \ g++-4.9 \ @@ -15,7 +15,7 @@ apt-get install -qq \ ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc -pip install tox --disable-pip-version-check +retry.py pip install tox --disable-pip-version-check ansible-test compile --color -v ansible-test sanity --color -v --tox --skip-test ansible-doc --python 2.7 diff --git a/test/utils/shippable/units.sh b/test/utils/shippable/units.sh index 653a8f33c4..f2c1b0b52f 100755 --- a/test/utils/shippable/units.sh +++ b/test/utils/shippable/units.sh @@ -2,16 +2,16 @@ set -o pipefail -add-apt-repository 'ppa:ubuntu-toolchain-r/test' -add-apt-repository 'ppa:fkrull/deadsnakes' +retry.py add-apt-repository 'ppa:ubuntu-toolchain-r/test' +retry.py add-apt-repository 'ppa:fkrull/deadsnakes' -apt-get update -qq -apt-get install -qq \ +retry.py apt-get update -qq +retry.py apt-get install -qq \ g++-4.9 \ python3.6-dev \ ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc -pip install tox --disable-pip-version-check +retry.py pip install tox --disable-pip-version-check ansible-test units --color -v --tox --coverage