From 036ba7eeec3062e629bd550c14370939e9b1eb8c Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 17 May 2017 13:49:04 +0800 Subject: [PATCH] Clean up Shippable tools and update job handling. - Tools are now in a tools subdirectory. - Removed obsolete ansible-core-ci tool. - Added run.py for starting new CI runs. - Improved handling of run IDs and URLs. - General code cleanup and docs updates. - Nightly CI runs use complete coverage. --- test/utils/shippable/ansible-core-ci | 375 ------------------- test/utils/shippable/shippable.sh | 5 + test/utils/shippable/{ => tools}/download.py | 51 ++- test/utils/shippable/tools/run.py | 138 +++++++ 4 files changed, 185 insertions(+), 384 deletions(-) delete mode 100755 test/utils/shippable/ansible-core-ci rename test/utils/shippable/{ => tools}/download.py (87%) create mode 100755 test/utils/shippable/tools/run.py diff --git a/test/utils/shippable/ansible-core-ci b/test/utils/shippable/ansible-core-ci deleted file mode 100755 index e91b426ad0..0000000000 --- a/test/utils/shippable/ansible-core-ci +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/env python - -# (c) 2016 Matt Clay -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import print_function - -import datetime -import json -import os -import subprocess -import sys -import traceback -import uuid - -from argparse import ArgumentParser -from textwrap import dedent -from time import sleep - - -def main(): - parser = ArgumentParser(description='Manage remote instances for testing.') - - parser.add_argument('-v', - '--verbose', - dest='verbose', - action='store_true', - help='write verbose output to stderr') - - parser.add_argument('--endpoint', - dest='endpoint', - default='https://14blg63h2i.execute-api.us-east-1.amazonaws.com', - help='api endpoint') - - parser.add_argument('--stage', - dest='stage', - default='prod', - choices=['dev', 'prod'], - help='api stage (default: prod)') - - subparsers = parser.add_subparsers() - - # sub parser: start - - start_parser = subparsers.add_parser('start', help='start instance') - start_parser.set_defaults(func=start_instance) - - start_subparsers = start_parser.add_subparsers(dest='start') - - start_parser.add_argument('platform', help='platform (ex: windows)') - start_parser.add_argument('version', help='version (ex: 2012-R2_RTM)') - - start_parser.add_argument('--id', - dest='instance_id', - default=uuid.uuid4(), - help='instance id to create') - - start_parser.add_argument('--public-key', - dest='ssh_key', - default=None, - help='path to ssh public key for authentication') - - start_parser.add_argument('--query', - dest='query', - action='store_true', - default=False, - help='query only, do not start instance') - - shippable = start_subparsers.add_parser('shippable', help='start instance for shippable testing') - - shippable.add_argument('--run-id', - dest='run', - default=os.environ.get('SHIPPABLE_BUILD_ID'), - help='shippable run id') - - shippable.add_argument('--job-number', - dest='job', - type=int, - default=os.environ.get('SHIPPABLE_JOB_NUMBER'), - help='shippable job number') - - remote_key = get_remote_key() - - remote = start_subparsers.add_parser('remote', help='start instance for remote testing') - - remote.add_argument('--key', - dest='key', - default=remote_key, - required=remote_key is None, - help='remote key') - - remote.add_argument('--nonce', - dest='nonce', - default=None, - help='optional nonce') - - # sub parser: get - - get_parser = subparsers.add_parser('get', help='get instance') - get_parser.set_defaults(func=get_instance) - - get_parser.add_argument('instance_id', help='id of instance previously created') - - get_parser.add_argument('--template', - dest='template', - help='inventory template') - - get_parser.add_argument('--tries', - dest='tries', - default=60, - type=int, - help='number of tries waiting for instance (default: 60)') - - get_parser.add_argument('--sleep', - dest='sleep', - default=10, - type=int, - help='sleep seconds between tries (default: 10)') - - # sub parser: stop - - stop_parser = subparsers.add_parser('stop', help='stop instance') - stop_parser.set_defaults(func=stop_instance) - - stop_parser.add_argument('instance_id', help='id of instance previously created') - - # parse - - args = parser.parse_args() - args.func(args) - - -def get_remote_key(): - path = os.path.join(os.environ['HOME'], '.ansible-core-ci.key') - - try: - with open(path, 'r') as f: - return f.read().strip() - except IOError: - return None - - -def get_instance_uri(args): - return '%s/%s/jobs/%s' % (args.endpoint, args.stage, args.instance_id) - - -def start_instance(args): - if args.ssh_key is None: - public_key = None - else: - with open(args.ssh_key, 'r') as f: - public_key = f.read() - - data = dict( - config=dict( - platform=args.platform, - version=args.version, - public_key=public_key, - query=args.query, - ) - ) - - if args.start == 'shippable': - auth = dict( - shippable=dict( - run_id=args.run, - job_number=args.job, - ), - ) - elif args.start == 'remote': - auth = dict( - remote=dict( - key=args.key, - nonce=args.nonce, - ), - ) - else: - raise Exception('auth required') - - data.update(dict(auth=auth)) - - if args.verbose: - print_stderr('starting instance: %s/%s (%s)' % (args.platform, args.version, args.instance_id)) - - headers = { - 'Content-Type': 'application/json', - } - - uri = get_instance_uri(args) - response = requests.put(uri, data=json.dumps(data), headers=headers) - - if response.status_code != 200: - raise Exception(create_http_error(response)) - - if args.verbose: - print_stderr('instance started: %s' % args.instance_id) - - print(args.instance_id) - - if args.query: - print_stderr(json.dumps(response.json(), indent=4, sort_keys=True)) - - -def stop_instance(args): - if args.verbose: - print_stderr('stopping instance: %s' % args.instance_id) - - uri = get_instance_uri(args) - response = requests.delete(uri) - - if response.status_code != 200: - raise Exception(create_http_error(response)) - - if args.verbose: - print_stderr('instance stopped: %s' % args.instance_id) - - -def get_instance(args): - if args.verbose: - print_stderr('waiting for instance: %s' % args.instance_id) - - uri = get_instance_uri(args) - start_time = datetime.datetime.utcnow() - - for i in range(args.tries): - response = requests.get(uri) - - if response.status_code != 200: - raise Exception(create_http_error(response)) - - response_json = response.json() - status = response_json['status'] - - if status == 'running': - end_time = datetime.datetime.utcnow() - duration = end_time - start_time - - if args.verbose: - print_stderr('waited %s for instance availability' % duration) - - connection = response_json['connection'] - inventory = make_inventory(args.template, connection, args.instance_id) - - print(inventory) - - return - - sleep(args.sleep) - - raise Exception('timeout waiting for instance') - - -def make_inventory(inventory_template, connection, instance_id): - if inventory_template is None: - template = dedent(''' - [winrm] - windows # @instance_id - - [winrm:vars] - ansible_connection=winrm - ansible_host=@ansible_host - ansible_user=@ansible_user - ansible_password=@ansible_password - ansible_port=5986 - ansible_winrm_server_cert_validation=ignore - ''').strip() - else: - with open(inventory_template, 'r') as f: - template = f.read() - - inventory = template\ - .replace('@instance_id', instance_id)\ - .replace('@ansible_host', connection['hostname'])\ - .replace('@ansible_port', str(connection.get('port', 22)))\ - .replace('@ansible_user', connection['username'])\ - .replace('@ansible_password', connection.get('password', '')) - - return inventory - - -def print_stderr(*args, **kwargs): - """Print to stderr.""" - - print(*args, file=sys.stderr, **kwargs) - - -def create_http_error(response): - response_json = response.json() - stack_trace = '' - - if 'message' in response_json: - message = response_json['message'] - elif 'errorMessage' in response_json: - message = response_json['errorMessage'].strip() - if 'stackTrace' in response_json: - trace = '\n'.join([x.rstrip() for x in traceback.format_list(response_json['stackTrace'])]) - stack_trace = ('\nTraceback (from remote server):\n%s' % trace) - else: - message = str(response_json) - - return '%s: %s%s' % (response.status_code, message, stack_trace) - - -class HttpRequest: - def __init__(self): - """ - primitive replacement for requests to avoid extra dependency - avoids use of urllib2 due to lack of SNI support - """ - - def get(self, url): - return self.request('GET', url) - - def delete(self, url): - return self.request('DELETE', url) - - def put(self, url, data=None, headers=None): - return self.request('PUT', url, data, headers) - - def request(self, method, url, data=None, headers=None): - args = ['/usr/bin/curl', '-s', '-S', '-i', '-X', method] - - if headers is None: - headers = {} - - headers['Expect'] = '' # don't send expect continue header - - for header in headers: - args += ['-H', '%s: %s' % (header, headers[header])] - - if data is not None: - args += ['-d', data] - - args += [url] - - header, body = subprocess.check_output(args).split('\r\n\r\n', 1) - - response_headers = header.split('\r\n') - first_line = response_headers[0] - http_response = first_line.split(' ') - status_code = int(http_response[1]) - - return HttpResponse(status_code, body) - - -class HttpResponse: - def __init__(self, status_code, response): - self.status_code = status_code - self.response = response - - def json(self): - try: - return json.loads(self.response) - except ValueError: - raise ValueError('Cannot parse response as JSON:\n%s' % self.response) - - -requests = HttpRequest() - -if __name__ == '__main__': - main() diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh index 84fc01785e..92b3700870 100755 --- a/test/utils/shippable/shippable.sh +++ b/test/utils/shippable/shippable.sh @@ -24,6 +24,11 @@ pip list --disable-pip-version-check export PATH="test/runner:${PATH}" export PYTHONIOENCODING='utf-8' +if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then + COVERAGE=yes + COMPLETE=yes +fi + if [ -n "${COVERAGE:-}" ]; then # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value export COVERAGE="--coverage" diff --git a/test/utils/shippable/download.py b/test/utils/shippable/tools/download.py similarity index 87% rename from test/utils/shippable/download.py rename to test/utils/shippable/tools/download.py index 16bec1c1df..a10fecd310 100755 --- a/test/utils/shippable/download.py +++ b/test/utils/shippable/tools/download.py @@ -17,16 +17,17 @@ # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . +"""CLI tool for downloading results from Shippable CI runs.""" from __future__ import print_function +# noinspection PyCompatibility +import argparse import json import os import re import requests -from argparse import ArgumentParser - try: import argcomplete except ImportError: @@ -34,9 +35,10 @@ except ImportError: def main(): + """Main program body.""" api_key = get_api_key() - parser = ArgumentParser(description='Download results from a Shippable run.') + parser = argparse.ArgumentParser(description='Download results from a Shippable run.') parser.add_argument('run_id', metavar='RUN', @@ -120,7 +122,9 @@ def main(): Authorization='apiToken %s' % args.api_key, ) - match = re.search(r'^https://app.shippable.com/github/(?P[^/]+)/(?P[^/]+)/runs/(?P[0-9]+)$', args.run_id) + match = re.search( + r'^https://app.shippable.com/github/(?P[^/]+)/(?P[^/]+)/runs/(?P[0-9]+)(?:/summary|(/(?P[0-9]+)))?$', + args.run_id) if not match: match = re.search(r'^(?P[^/]+)/(?P[^/]+)/(?P[0-9]+)$', args.run_id) @@ -129,6 +133,13 @@ def main(): account = match.group('account') project = match.group('project') run_number = int(match.group('run_number')) + job_number = int(match.group('job_number')) if match.group('job_number') else None + + if job_number: + if args.job_number: + exit('ERROR: job number found in url and specified with --job-number') + + args.job_number = [job_number] url = 'https://api.shippable.com/projects' response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers) @@ -148,7 +159,7 @@ def main(): run = [run for run in response.json() if run['runNumber'] == run_number][0] args.run_id = run['id'] - else: + elif re.search('^[a-f0-9]+$', args.run_id): url = 'https://api.shippable.com/runs/%s' % args.run_id response = requests.get(url, headers=headers) @@ -161,6 +172,8 @@ def main(): account = run['subscriptionOrgName'] project = run['projectName'] run_number = run['runNumber'] + else: + exit('ERROR: invalid run: %s' % args.run_id) output_dir = '%s/%s/%s' % (account, project, run_number) @@ -228,6 +241,11 @@ def main(): def extract_contents(args, path, output_dir): + """ + :type args: any + :type path: str + :type output_dir: str + """ if not args.test: if not os.path.exists(path): return @@ -256,6 +274,13 @@ def extract_contents(args, path, output_dir): def download(args, headers, path, url, is_json=True): + """ + :type args: any + :type headers: dict[str, str] + :type path: str + :type url: str + :type is_json: bool + """ if args.verbose or args.test: print(path) @@ -278,16 +303,24 @@ def download(args, headers, path, url, is_json=True): if not os.path.exists(directory): os.makedirs(directory) - with open(path, 'w') as f: - f.write(content) + with open(path, 'w') as content_fd: + content_fd.write(content) def get_api_key(): + """ + rtype: str + """ + key = os.environ.get('SHIPPABLE_KEY', None) + + if key: + return key + path = os.path.join(os.environ['HOME'], '.shippable.key') try: - with open(path, 'r') as f: - return f.read().strip() + with open(path, 'r') as key_fd: + return key_fd.read().strip() except IOError: return None diff --git a/test/utils/shippable/tools/run.py b/test/utils/shippable/tools/run.py new file mode 100755 index 0000000000..7eb60bcda1 --- /dev/null +++ b/test/utils/shippable/tools/run.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK + +# (c) 2016 Red Hat, Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +"""CLI tool for starting new Shippable CI runs.""" + +from __future__ import print_function + +# noinspection PyCompatibility +import argparse +import json +import os +import requests + +try: + import argcomplete +except ImportError: + argcomplete = None + + +def main(): + """Main program body.""" + api_key = get_api_key() + + parser = argparse.ArgumentParser(description='Start a new Shippable run.') + + parser.add_argument('project', + metavar='account/project', + help='Shippable account/project') + + target = parser.add_mutually_exclusive_group() + + target.add_argument('--branch', + help='branch name') + + target.add_argument('--run', + metavar='ID', + help='Shippable run ID') + + parser.add_argument('--key', + metavar='KEY', + default=api_key, + required=not api_key, + help='Shippable API key') + + parser.add_argument('--env', + nargs=2, + metavar=('KEY', 'VALUE'), + action='append', + help='environment variable to pass') + + if argcomplete: + argcomplete.autocomplete(parser) + + args = parser.parse_args() + + headers = dict( + Authorization='apiToken %s' % args.key, + ) + + # get project ID + + data = dict( + projectFullNames=args.project, + ) + + url = 'https://api.shippable.com/projects' + response = requests.get(url, data, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + result = response.json() + + if len(result) != 1: + raise Exception( + 'Received %d items instead of 1 looking for %s in:\n%s' % ( + len(result), + args.project, + json.dumps(result, indent=4, sort_keys=True))) + + project_id = response.json()[0]['id'] + + # new build + + data = dict( + globalEnv=['%s=%s' % (kp[0], kp[1]) for kp in args.env or []] + ) + + if args.branch: + data['branch'] = args.branch + elif args.run: + data['runId'] = args.run + + url = 'https://api.shippable.com/projects/%s/newBuild' % project_id + response = requests.post(url, data, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + print(json.dumps(response.json(), indent=4, sort_keys=True)) + + +def get_api_key(): + """ + rtype: str + """ + key = os.environ.get('SHIPPABLE_KEY', None) + + if key: + return key + + path = os.path.join(os.environ['HOME'], '.shippable.key') + + try: + with open(path, 'r') as key_fd: + return key_fd.read().strip() + except IOError: + return None + + +if __name__ == '__main__': + main()