mirror of
https://github.com/ansible-collections/community.general.git
synced 2024-09-14 20:13:21 +02:00
Clean up Shippable tools and update job handling.
- Tools are now in a tools subdirectory. - Removed obsolete ansible-core-ci tool. - Added run.py for starting new CI runs. - Improved handling of run IDs and URLs. - General code cleanup and docs updates. - Nightly CI runs use complete coverage.
This commit is contained in:
Matt Clay
parent
8937246f8f
commit
036ba7eeec
4 changed files with 185 additions and 384 deletions
|
|
@ -1,375 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# (c) 2016 Matt Clay <matt@mystile.com>
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
import uuid
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from textwrap import dedent
|
||||
from time import sleep
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser(description='Manage remote instances for testing.')
|
||||
|
||||
parser.add_argument('-v',
|
||||
'--verbose',
|
||||
dest='verbose',
|
||||
action='store_true',
|
||||
help='write verbose output to stderr')
|
||||
|
||||
parser.add_argument('--endpoint',
|
||||
dest='endpoint',
|
||||
default='https://14blg63h2i.execute-api.us-east-1.amazonaws.com',
|
||||
help='api endpoint')
|
||||
|
||||
parser.add_argument('--stage',
|
||||
dest='stage',
|
||||
default='prod',
|
||||
choices=['dev', 'prod'],
|
||||
help='api stage (default: prod)')
|
||||
|
||||
subparsers = parser.add_subparsers()
|
||||
|
||||
# sub parser: start
|
||||
|
||||
start_parser = subparsers.add_parser('start', help='start instance')
|
||||
start_parser.set_defaults(func=start_instance)
|
||||
|
||||
start_subparsers = start_parser.add_subparsers(dest='start')
|
||||
|
||||
start_parser.add_argument('platform', help='platform (ex: windows)')
|
||||
start_parser.add_argument('version', help='version (ex: 2012-R2_RTM)')
|
||||
|
||||
start_parser.add_argument('--id',
|
||||
dest='instance_id',
|
||||
default=uuid.uuid4(),
|
||||
help='instance id to create')
|
||||
|
||||
start_parser.add_argument('--public-key',
|
||||
dest='ssh_key',
|
||||
default=None,
|
||||
help='path to ssh public key for authentication')
|
||||
|
||||
start_parser.add_argument('--query',
|
||||
dest='query',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='query only, do not start instance')
|
||||
|
||||
shippable = start_subparsers.add_parser('shippable', help='start instance for shippable testing')
|
||||
|
||||
shippable.add_argument('--run-id',
|
||||
dest='run',
|
||||
default=os.environ.get('SHIPPABLE_BUILD_ID'),
|
||||
help='shippable run id')
|
||||
|
||||
shippable.add_argument('--job-number',
|
||||
dest='job',
|
||||
type=int,
|
||||
default=os.environ.get('SHIPPABLE_JOB_NUMBER'),
|
||||
help='shippable job number')
|
||||
|
||||
remote_key = get_remote_key()
|
||||
|
||||
remote = start_subparsers.add_parser('remote', help='start instance for remote testing')
|
||||
|
||||
remote.add_argument('--key',
|
||||
dest='key',
|
||||
default=remote_key,
|
||||
required=remote_key is None,
|
||||
help='remote key')
|
||||
|
||||
remote.add_argument('--nonce',
|
||||
dest='nonce',
|
||||
default=None,
|
||||
help='optional nonce')
|
||||
|
||||
# sub parser: get
|
||||
|
||||
get_parser = subparsers.add_parser('get', help='get instance')
|
||||
get_parser.set_defaults(func=get_instance)
|
||||
|
||||
get_parser.add_argument('instance_id', help='id of instance previously created')
|
||||
|
||||
get_parser.add_argument('--template',
|
||||
dest='template',
|
||||
help='inventory template')
|
||||
|
||||
get_parser.add_argument('--tries',
|
||||
dest='tries',
|
||||
default=60,
|
||||
type=int,
|
||||
help='number of tries waiting for instance (default: 60)')
|
||||
|
||||
get_parser.add_argument('--sleep',
|
||||
dest='sleep',
|
||||
default=10,
|
||||
type=int,
|
||||
help='sleep seconds between tries (default: 10)')
|
||||
|
||||
# sub parser: stop
|
||||
|
||||
stop_parser = subparsers.add_parser('stop', help='stop instance')
|
||||
stop_parser.set_defaults(func=stop_instance)
|
||||
|
||||
stop_parser.add_argument('instance_id', help='id of instance previously created')
|
||||
|
||||
# parse
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
|
||||
def get_remote_key():
|
||||
path = os.path.join(os.environ['HOME'], '.ansible-core-ci.key')
|
||||
|
||||
try:
|
||||
with open(path, 'r') as f:
|
||||
return f.read().strip()
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
|
||||
def get_instance_uri(args):
|
||||
return '%s/%s/jobs/%s' % (args.endpoint, args.stage, args.instance_id)
|
||||
|
||||
|
||||
def start_instance(args):
|
||||
if args.ssh_key is None:
|
||||
public_key = None
|
||||
else:
|
||||
with open(args.ssh_key, 'r') as f:
|
||||
public_key = f.read()
|
||||
|
||||
data = dict(
|
||||
config=dict(
|
||||
platform=args.platform,
|
||||
version=args.version,
|
||||
public_key=public_key,
|
||||
query=args.query,
|
||||
)
|
||||
)
|
||||
|
||||
if args.start == 'shippable':
|
||||
auth = dict(
|
||||
shippable=dict(
|
||||
run_id=args.run,
|
||||
job_number=args.job,
|
||||
),
|
||||
)
|
||||
elif args.start == 'remote':
|
||||
auth = dict(
|
||||
remote=dict(
|
||||
key=args.key,
|
||||
nonce=args.nonce,
|
||||
),
|
||||
)
|
||||
else:
|
||||
raise Exception('auth required')
|
||||
|
||||
data.update(dict(auth=auth))
|
||||
|
||||
if args.verbose:
|
||||
print_stderr('starting instance: %s/%s (%s)' % (args.platform, args.version, args.instance_id))
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
|
||||
uri = get_instance_uri(args)
|
||||
response = requests.put(uri, data=json.dumps(data), headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(create_http_error(response))
|
||||
|
||||
if args.verbose:
|
||||
print_stderr('instance started: %s' % args.instance_id)
|
||||
|
||||
print(args.instance_id)
|
||||
|
||||
if args.query:
|
||||
print_stderr(json.dumps(response.json(), indent=4, sort_keys=True))
|
||||
|
||||
|
||||
def stop_instance(args):
|
||||
if args.verbose:
|
||||
print_stderr('stopping instance: %s' % args.instance_id)
|
||||
|
||||
uri = get_instance_uri(args)
|
||||
response = requests.delete(uri)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(create_http_error(response))
|
||||
|
||||
if args.verbose:
|
||||
print_stderr('instance stopped: %s' % args.instance_id)
|
||||
|
||||
|
||||
def get_instance(args):
|
||||
if args.verbose:
|
||||
print_stderr('waiting for instance: %s' % args.instance_id)
|
||||
|
||||
uri = get_instance_uri(args)
|
||||
start_time = datetime.datetime.utcnow()
|
||||
|
||||
for i in range(args.tries):
|
||||
response = requests.get(uri)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(create_http_error(response))
|
||||
|
||||
response_json = response.json()
|
||||
status = response_json['status']
|
||||
|
||||
if status == 'running':
|
||||
end_time = datetime.datetime.utcnow()
|
||||
duration = end_time - start_time
|
||||
|
||||
if args.verbose:
|
||||
print_stderr('waited %s for instance availability' % duration)
|
||||
|
||||
connection = response_json['connection']
|
||||
inventory = make_inventory(args.template, connection, args.instance_id)
|
||||
|
||||
print(inventory)
|
||||
|
||||
return
|
||||
|
||||
sleep(args.sleep)
|
||||
|
||||
raise Exception('timeout waiting for instance')
|
||||
|
||||
|
||||
def make_inventory(inventory_template, connection, instance_id):
|
||||
if inventory_template is None:
|
||||
template = dedent('''
|
||||
[winrm]
|
||||
windows # @instance_id
|
||||
|
||||
[winrm:vars]
|
||||
ansible_connection=winrm
|
||||
ansible_host=@ansible_host
|
||||
ansible_user=@ansible_user
|
||||
ansible_password=@ansible_password
|
||||
ansible_port=5986
|
||||
ansible_winrm_server_cert_validation=ignore
|
||||
''').strip()
|
||||
else:
|
||||
with open(inventory_template, 'r') as f:
|
||||
template = f.read()
|
||||
|
||||
inventory = template\
|
||||
.replace('@instance_id', instance_id)\
|
||||
.replace('@ansible_host', connection['hostname'])\
|
||||
.replace('@ansible_port', str(connection.get('port', 22)))\
|
||||
.replace('@ansible_user', connection['username'])\
|
||||
.replace('@ansible_password', connection.get('password', ''))
|
||||
|
||||
return inventory
|
||||
|
||||
|
||||
def print_stderr(*args, **kwargs):
|
||||
"""Print to stderr."""
|
||||
|
||||
print(*args, file=sys.stderr, **kwargs)
|
||||
|
||||
|
||||
def create_http_error(response):
|
||||
response_json = response.json()
|
||||
stack_trace = ''
|
||||
|
||||
if 'message' in response_json:
|
||||
message = response_json['message']
|
||||
elif 'errorMessage' in response_json:
|
||||
message = response_json['errorMessage'].strip()
|
||||
if 'stackTrace' in response_json:
|
||||
trace = '\n'.join([x.rstrip() for x in traceback.format_list(response_json['stackTrace'])])
|
||||
stack_trace = ('\nTraceback (from remote server):\n%s' % trace)
|
||||
else:
|
||||
message = str(response_json)
|
||||
|
||||
return '%s: %s%s' % (response.status_code, message, stack_trace)
|
||||
|
||||
|
||||
class HttpRequest:
|
||||
def __init__(self):
|
||||
"""
|
||||
primitive replacement for requests to avoid extra dependency
|
||||
avoids use of urllib2 due to lack of SNI support
|
||||
"""
|
||||
|
||||
def get(self, url):
|
||||
return self.request('GET', url)
|
||||
|
||||
def delete(self, url):
|
||||
return self.request('DELETE', url)
|
||||
|
||||
def put(self, url, data=None, headers=None):
|
||||
return self.request('PUT', url, data, headers)
|
||||
|
||||
def request(self, method, url, data=None, headers=None):
|
||||
args = ['/usr/bin/curl', '-s', '-S', '-i', '-X', method]
|
||||
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
headers['Expect'] = '' # don't send expect continue header
|
||||
|
||||
for header in headers:
|
||||
args += ['-H', '%s: %s' % (header, headers[header])]
|
||||
|
||||
if data is not None:
|
||||
args += ['-d', data]
|
||||
|
||||
args += [url]
|
||||
|
||||
header, body = subprocess.check_output(args).split('\r\n\r\n', 1)
|
||||
|
||||
response_headers = header.split('\r\n')
|
||||
first_line = response_headers[0]
|
||||
http_response = first_line.split(' ')
|
||||
status_code = int(http_response[1])
|
||||
|
||||
return HttpResponse(status_code, body)
|
||||
|
||||
|
||||
class HttpResponse:
|
||||
def __init__(self, status_code, response):
|
||||
self.status_code = status_code
|
||||
self.response = response
|
||||
|
||||
def json(self):
|
||||
try:
|
||||
return json.loads(self.response)
|
||||
except ValueError:
|
||||
raise ValueError('Cannot parse response as JSON:\n%s' % self.response)
|
||||
|
||||
|
||||
requests = HttpRequest()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -24,6 +24,11 @@ pip list --disable-pip-version-check
|
|||
export PATH="test/runner:${PATH}"
|
||||
export PYTHONIOENCODING='utf-8'
|
||||
|
||||
if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then
|
||||
COVERAGE=yes
|
||||
COMPLETE=yes
|
||||
fi
|
||||
|
||||
if [ -n "${COVERAGE:-}" ]; then
|
||||
# on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
|
||||
export COVERAGE="--coverage"
|
||||
|
|
|
|||
|
|
@ -17,16 +17,17 @@
|
|||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""CLI tool for downloading results from Shippable CI runs."""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
# noinspection PyCompatibility
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
try:
|
||||
import argcomplete
|
||||
except ImportError:
|
||||
|
|
@ -34,9 +35,10 @@ except ImportError:
|
|||
|
||||
|
||||
def main():
|
||||
"""Main program body."""
|
||||
api_key = get_api_key()
|
||||
|
||||
parser = ArgumentParser(description='Download results from a Shippable run.')
|
||||
parser = argparse.ArgumentParser(description='Download results from a Shippable run.')
|
||||
|
||||
parser.add_argument('run_id',
|
||||
metavar='RUN',
|
||||
|
|
@ -120,7 +122,9 @@ def main():
|
|||
Authorization='apiToken %s' % args.api_key,
|
||||
)
|
||||
|
||||
match = re.search(r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)$', args.run_id)
|
||||
match = re.search(
|
||||
r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)(?:/summary|(/(?P<job_number>[0-9]+)))?$',
|
||||
args.run_id)
|
||||
|
||||
if not match:
|
||||
match = re.search(r'^(?P<account>[^/]+)/(?P<project>[^/]+)/(?P<run_number>[0-9]+)$', args.run_id)
|
||||
|
|
@ -129,6 +133,13 @@ def main():
|
|||
account = match.group('account')
|
||||
project = match.group('project')
|
||||
run_number = int(match.group('run_number'))
|
||||
job_number = int(match.group('job_number')) if match.group('job_number') else None
|
||||
|
||||
if job_number:
|
||||
if args.job_number:
|
||||
exit('ERROR: job number found in url and specified with --job-number')
|
||||
|
||||
args.job_number = [job_number]
|
||||
|
||||
url = 'https://api.shippable.com/projects'
|
||||
response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers)
|
||||
|
|
@ -148,7 +159,7 @@ def main():
|
|||
run = [run for run in response.json() if run['runNumber'] == run_number][0]
|
||||
|
||||
args.run_id = run['id']
|
||||
else:
|
||||
elif re.search('^[a-f0-9]+$', args.run_id):
|
||||
url = 'https://api.shippable.com/runs/%s' % args.run_id
|
||||
|
||||
response = requests.get(url, headers=headers)
|
||||
|
|
@ -161,6 +172,8 @@ def main():
|
|||
account = run['subscriptionOrgName']
|
||||
project = run['projectName']
|
||||
run_number = run['runNumber']
|
||||
else:
|
||||
exit('ERROR: invalid run: %s' % args.run_id)
|
||||
|
||||
output_dir = '%s/%s/%s' % (account, project, run_number)
|
||||
|
||||
|
|
@ -228,6 +241,11 @@ def main():
|
|||
|
||||
|
||||
def extract_contents(args, path, output_dir):
|
||||
"""
|
||||
:type args: any
|
||||
:type path: str
|
||||
:type output_dir: str
|
||||
"""
|
||||
if not args.test:
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
|
@ -256,6 +274,13 @@ def extract_contents(args, path, output_dir):
|
|||
|
||||
|
||||
def download(args, headers, path, url, is_json=True):
|
||||
"""
|
||||
:type args: any
|
||||
:type headers: dict[str, str]
|
||||
:type path: str
|
||||
:type url: str
|
||||
:type is_json: bool
|
||||
"""
|
||||
if args.verbose or args.test:
|
||||
print(path)
|
||||
|
||||
|
|
@ -278,16 +303,24 @@ def download(args, headers, path, url, is_json=True):
|
|||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
|
||||
with open(path, 'w') as f:
|
||||
f.write(content)
|
||||
with open(path, 'w') as content_fd:
|
||||
content_fd.write(content)
|
||||
|
||||
|
||||
def get_api_key():
|
||||
"""
|
||||
rtype: str
|
||||
"""
|
||||
key = os.environ.get('SHIPPABLE_KEY', None)
|
||||
|
||||
if key:
|
||||
return key
|
||||
|
||||
path = os.path.join(os.environ['HOME'], '.shippable.key')
|
||||
|
||||
try:
|
||||
with open(path, 'r') as f:
|
||||
return f.read().strip()
|
||||
with open(path, 'r') as key_fd:
|
||||
return key_fd.read().strip()
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
138
test/utils/shippable/tools/run.py
Executable file
138
test/utils/shippable/tools/run.py
Executable file
|
|
@ -0,0 +1,138 @@
|
|||
#!/usr/bin/env python
|
||||
# PYTHON_ARGCOMPLETE_OK
|
||||
|
||||
# (c) 2016 Red Hat, Inc.
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""CLI tool for starting new Shippable CI runs."""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
# noinspection PyCompatibility
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
|
||||
try:
|
||||
import argcomplete
|
||||
except ImportError:
|
||||
argcomplete = None
|
||||
|
||||
|
||||
def main():
|
||||
"""Main program body."""
|
||||
api_key = get_api_key()
|
||||
|
||||
parser = argparse.ArgumentParser(description='Start a new Shippable run.')
|
||||
|
||||
parser.add_argument('project',
|
||||
metavar='account/project',
|
||||
help='Shippable account/project')
|
||||
|
||||
target = parser.add_mutually_exclusive_group()
|
||||
|
||||
target.add_argument('--branch',
|
||||
help='branch name')
|
||||
|
||||
target.add_argument('--run',
|
||||
metavar='ID',
|
||||
help='Shippable run ID')
|
||||
|
||||
parser.add_argument('--key',
|
||||
metavar='KEY',
|
||||
default=api_key,
|
||||
required=not api_key,
|
||||
help='Shippable API key')
|
||||
|
||||
parser.add_argument('--env',
|
||||
nargs=2,
|
||||
metavar=('KEY', 'VALUE'),
|
||||
action='append',
|
||||
help='environment variable to pass')
|
||||
|
||||
if argcomplete:
|
||||
argcomplete.autocomplete(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
headers = dict(
|
||||
Authorization='apiToken %s' % args.key,
|
||||
)
|
||||
|
||||
# get project ID
|
||||
|
||||
data = dict(
|
||||
projectFullNames=args.project,
|
||||
)
|
||||
|
||||
url = 'https://api.shippable.com/projects'
|
||||
response = requests.get(url, data, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
|
||||
result = response.json()
|
||||
|
||||
if len(result) != 1:
|
||||
raise Exception(
|
||||
'Received %d items instead of 1 looking for %s in:\n%s' % (
|
||||
len(result),
|
||||
args.project,
|
||||
json.dumps(result, indent=4, sort_keys=True)))
|
||||
|
||||
project_id = response.json()[0]['id']
|
||||
|
||||
# new build
|
||||
|
||||
data = dict(
|
||||
globalEnv=['%s=%s' % (kp[0], kp[1]) for kp in args.env or []]
|
||||
)
|
||||
|
||||
if args.branch:
|
||||
data['branch'] = args.branch
|
||||
elif args.run:
|
||||
data['runId'] = args.run
|
||||
|
||||
url = 'https://api.shippable.com/projects/%s/newBuild' % project_id
|
||||
response = requests.post(url, data, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
|
||||
print(json.dumps(response.json(), indent=4, sort_keys=True))
|
||||
|
||||
|
||||
def get_api_key():
|
||||
"""
|
||||
rtype: str
|
||||
"""
|
||||
key = os.environ.get('SHIPPABLE_KEY', None)
|
||||
|
||||
if key:
|
||||
return key
|
||||
|
||||
path = os.path.join(os.environ['HOME'], '.shippable.key')
|
||||
|
||||
try:
|
||||
with open(path, 'r') as key_fd:
|
||||
return key_fd.read().strip()
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in a new issue