From 2cc4a372c5b65f37e2c4e24d293714c2d851136c Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Sat, 29 Mar 2014 17:27:48 -0700 Subject: [PATCH 1/7] Adds integration tests for gce module. --- test/integration/Makefile | 9 ++ test/integration/cleanup_gce.py | 102 ++++++++++++++++++ test/integration/credentials.template | 5 + test/integration/gce.yml | 5 + .../roles/test_gce/defaults/main.yml | 6 ++ .../integration/roles/test_gce/tasks/main.yml | 97 +++++++++++++++++ 6 files changed, 224 insertions(+) create mode 100644 test/integration/cleanup_gce.py create mode 100644 test/integration/gce.yml create mode 100644 test/integration/roles/test_gce/defaults/main.yml create mode 100644 test/integration/roles/test_gce/tasks/main.yml diff --git a/test/integration/Makefile b/test/integration/Makefile index da2758c140..d8ca0a10c2 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -39,6 +39,9 @@ cloud_cleanup: amazon_cleanup rackspace_cleanup amazon_cleanup: python cleanup_ec2.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" +gce_cleanup: + python cleanup_gce.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" + rackspace_cleanup: @echo "FIXME - cleanup_rax.py not yet implemented" @# python cleanup_rax.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" @@ -53,6 +56,12 @@ amazon: $(CREDENTIALS_FILE) CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make amazon_cleanup ; \ exit $$RC; +gce: $(CREDENTIALS_FILE) + ansible-playbook gce.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -e "resource_prefix=$(CLOUD_RESOURCE_PREFIX)" -v $(TEST_FLAGS) ; \ + RC=$$? ; \ + CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make gce_cleanup ; \ + exit $$RC; + rackspace: $(CREDENTIALS_FILE) ansible-playbook rackspace.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -e "resource_prefix=$(CLOUD_RESOURCE_PREFIX)" -v $(TEST_FLAGS) ; \ RC=$$? ; \ diff --git a/test/integration/cleanup_gce.py b/test/integration/cleanup_gce.py new file mode 100644 index 0000000000..9b1645e93e --- /dev/null +++ b/test/integration/cleanup_gce.py @@ -0,0 +1,102 @@ +''' +Find and delete GCE resources matching the provided --match string. Unless +--yes|-y is provided, the prompt for confirmation prior to deleting resources. +Please use caution, you can easily delete your *ENTIRE* GCE infrastructure. +''' + +import os +import re +import sys +import optparse +import yaml + +try: + from libcloud.compute.types import Provider + from libcloud.compute.providers import get_driver + from libcloud.common.google import GoogleBaseError, QuotaExceededError, \ + ResourceExistsError, ResourceInUseError, ResourceNotFoundError + _ = Provider.GCE +except ImportError: + print("failed=True " + \ + "msg='libcloud with GCE support (0.13.3+) required for this module'") + sys.exit(1) + + +def delete_gce_resources(get_func, attr, opts): + for item in get_func(): + val = getattr(item, attr) + if re.search(opts.match_re, val, re.IGNORECASE): + prompt_and_delete(item, "Delete matching %s? [y/n]: " % (item,), opts.assumeyes) + +def prompt_and_delete(item, prompt, assumeyes): + if not assumeyes: + assumeyes = raw_input(prompt).lower() == 'y' + assert hasattr(item, 'destroy'), "Class <%s> has no delete attribute" % item.__class__ + if assumeyes: + item.destroy() + print ("Deleted %s" % item) + +def parse_args(): + default_service_account_email = None + default_pem_file = None + default_project_id = None + + # Load details from credentials.yml + if os.path.isfile('credentials.yml'): + credentials = yaml.load(open('credentials.yml', 'r')) + + if default_service_account_email is None: + default_service_account_email = credentials['gce_service_account_email'] + if default_pem_file is None: + default_pem_file = credentials['gce_pem_file'] + if default_project_id is None: + default_project_id = credentials['gce_project_id'] + + parser = optparse.OptionParser(usage="%s [options]" % (sys.argv[0],), + description=__doc__) + parser.add_option("--service_account_email", + action="store", dest="service_account_email", + default=default_service_account_email, + help="GCE service account email. Default is loaded from credentials.yml.") + parser.add_option("--pem_file", + action="store", dest="pem_file", + default=default_pem_file, + help="GCE client key. Default is loaded from credentials.yml.") + parser.add_option("--project_id", + action="store", dest="project_id", + default=default_project_id, + help="Google Cloud project ID. Default is loaded from credentials.yml.") + parser.add_option("--credentials", "-c", + action="store", dest="credential_file", + default="credentials.yml", + help="YAML file to read cloud credentials (default: %default)") + parser.add_option("--yes", "-y", + action="store_true", dest="assumeyes", + default=False, + help="Don't prompt for confirmation") + parser.add_option("--match", + action="store", dest="match_re", + default="^ansible-testing-", + help="Regular expression used to find GCE resources (default: %default)") + + (opts, args) = parser.parse_args() + for required in ['service_account_email', 'pem_file', 'project_id']: + if getattr(opts, required) is None: + parser.error("Missing required parameter: --%s" % required) + + return (opts, args) + +if __name__ == '__main__': + + (opts, args) = parse_args() + + # Connect to GCE + gce_cls = get_driver(Provider.GCE) + gce = gce_cls( + opts.service_account_email, opts.pem_file, project=opts.project_id) + + try: + # Delete matching instances + delete_gce_resources(gce.list_nodes, 'name', opts) + except KeyboardInterrupt, e: + print "\nExiting on user command." diff --git a/test/integration/credentials.template b/test/integration/credentials.template index f21100405f..12316254bb 100644 --- a/test/integration/credentials.template +++ b/test/integration/credentials.template @@ -3,5 +3,10 @@ ec2_access_key: ec2_secret_key: +# GCE Credentials +service_account_email: +pem_file: +project_id: + # GITHUB SSH private key - a path to a SSH private key for use with github.com github_ssh_private_key: "{{ lookup('env','HOME') }}/.ssh/id_rsa" diff --git a/test/integration/gce.yml b/test/integration/gce.yml new file mode 100644 index 0000000000..75287abab1 --- /dev/null +++ b/test/integration/gce.yml @@ -0,0 +1,5 @@ +- hosts: testhost + gather_facts: true + roles: + - { role: test_gce, tags: test_gce } + # TODO: tests for gce_net, gce_pd, etc. diff --git a/test/integration/roles/test_gce/defaults/main.yml b/test/integration/roles/test_gce/defaults/main.yml new file mode 100644 index 0000000000..1564808d79 --- /dev/null +++ b/test/integration/roles/test_gce/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# defaults file for test_gce +instance_name: "{{ resource_prefix|lower }}" +service_account_email: "{{ gce_service_account_email }}" +pem_file: "{{ gce_pem_file }}" +project_id: "{{ gce_project_id }}" diff --git a/test/integration/roles/test_gce/tasks/main.yml b/test/integration/roles/test_gce/tasks/main.yml new file mode 100644 index 0000000000..8791c281a5 --- /dev/null +++ b/test/integration/roles/test_gce/tasks/main.yml @@ -0,0 +1,97 @@ +# TODO: lots of attributes not covered: machine_type, zone, metadata, tags, etc. +# +# ============================================================ +- name: test with no parameters + gce: + register: result + ignore_errors: true + +- name: assert failure when called with no parameters + assert: + that: + - 'result.failed' + - 'result.msg == "Missing GCE connection parameters in libcloud secrets file."' + +# ============================================================ +- name: test missing name + gce: + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + register: result + ignore_errors: true + +- name: assert failure when called with no parameters + assert: + that: + - 'result.failed' + - 'result.msg == "Must specify a \"name\" or \"instance_names\""' + +# ============================================================ +- name: test state=present (expected changed=true) + gce: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert state=present (expected changed=true) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "present"' + +# ============================================================ +- name: test state=present (expected changed=false) + gce: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert state=present (expected changed=false) + assert: + that: + - 'not result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "present"' + +# ============================================================ +- name: test state=absent (expected changed=true) + gce: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert state=absent (expected changed=true) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' + +# ============================================================ +- name: test state=absent (expected changed=false) + gce: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert state=absent (expected changed=false) + assert: + that: + - 'not result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' + From 1e76fe2569523547669dddb068dafd71ea1d771b Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Sun, 30 Mar 2014 14:49:55 -0700 Subject: [PATCH 2/7] Adds integration tests for gce_pd module. --- test/integration/cleanup_gce.py | 2 + test/integration/gce.yml | 3 +- .../roles/test_gce_pd/defaults/main.yml | 6 + .../roles/test_gce_pd/tasks/main.yml | 126 ++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 test/integration/roles/test_gce_pd/defaults/main.yml create mode 100644 test/integration/roles/test_gce_pd/tasks/main.yml diff --git a/test/integration/cleanup_gce.py b/test/integration/cleanup_gce.py index 9b1645e93e..58b74ca324 100644 --- a/test/integration/cleanup_gce.py +++ b/test/integration/cleanup_gce.py @@ -98,5 +98,7 @@ if __name__ == '__main__': try: # Delete matching instances delete_gce_resources(gce.list_nodes, 'name', opts) + # Delete matching disks + delete_gce_resources(gce.list_volumes, 'name', opts) except KeyboardInterrupt, e: print "\nExiting on user command." diff --git a/test/integration/gce.yml b/test/integration/gce.yml index 75287abab1..6e314315c5 100644 --- a/test/integration/gce.yml +++ b/test/integration/gce.yml @@ -2,4 +2,5 @@ gather_facts: true roles: - { role: test_gce, tags: test_gce } - # TODO: tests for gce_net, gce_pd, etc. + - { role: test_gce_pd, tags: test_gce_pd } + # TODO: tests for gce_lb, gce_net, gc_storage diff --git a/test/integration/roles/test_gce_pd/defaults/main.yml b/test/integration/roles/test_gce_pd/defaults/main.yml new file mode 100644 index 0000000000..1564808d79 --- /dev/null +++ b/test/integration/roles/test_gce_pd/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# defaults file for test_gce +instance_name: "{{ resource_prefix|lower }}" +service_account_email: "{{ gce_service_account_email }}" +pem_file: "{{ gce_pem_file }}" +project_id: "{{ gce_project_id }}" diff --git a/test/integration/roles/test_gce_pd/tasks/main.yml b/test/integration/roles/test_gce_pd/tasks/main.yml new file mode 100644 index 0000000000..e7c7f15fbe --- /dev/null +++ b/test/integration/roles/test_gce_pd/tasks/main.yml @@ -0,0 +1,126 @@ +# TODO: need tests for read/write mode. + +# ============================================================ +- name: test missing name + gce_pd: + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + register: result + ignore_errors: true + +- name: assert failure when called with no parameters + assert: + that: + - 'result.failed' + - 'result.msg == "missing required arguments: name"' + +# ============================================================ +- name: test state=present (expected changed=true) + gce_pd: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert state=present (expected changed=true) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.size_gb == 10' # default size + - 'result.zone == "us-central1-b"' # default zone + - 'result.state == "present"' + +# ============================================================ +- name: test state=present (expected changed=false) + gce_pd: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert state=present (expected changed=false) + assert: + that: + - 'not result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "present"' + +# ============================================================ +- name: test state=absent (expected changed=true) + gce_pd: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert state=absent (expected changed=true) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' + +# ============================================================ +- name: test state=absent (expected changed=false) + gce_pd: + name: "{{ instance_name }}" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert state=absent (expected changed=false) + assert: + that: + - 'not result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' + +# ============================================================ +- name: test non-default size/zone + gce_pd: + name: "{{ instance_name }}" + size_gb: 5 + zone: us-central1-a + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert non-default size/zone + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.size_gb == 5' + - 'result.zone == "us-central1-a"' + - 'result.state == "present"' + +# ============================================================ +- name: test non-default size/zone (state=absent) + gce_pd: + name: "{{ instance_name }}" + size_gb: 5 + zone: us-central1-a + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert non-default size/zone (state=absent) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' From 0b45b1256df5c7d91faf187d7a6ecb270d36b31a Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Fri, 11 Apr 2014 14:40:52 -0700 Subject: [PATCH 3/7] Adds support for creating GCE persistent disks from images --- library/cloud/gce_pd | 14 ++++++- .../roles/test_gce_pd/tasks/main.yml | 37 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/library/cloud/gce_pd b/library/cloud/gce_pd index e5ea6cc4ad..98e8e8d15e 100644 --- a/library/cloud/gce_pd +++ b/library/cloud/gce_pd @@ -62,6 +62,12 @@ options: required: false default: 10 aliases: [] + image: + description: + - the source image to use for the disk + required: false + default: null + aliases: [] state: description: - desired state of the persistent disk @@ -132,6 +138,7 @@ def main(): mode = dict(default='READ_ONLY', choices=['READ_WRITE', 'READ_ONLY']), name = dict(required=True), size_gb = dict(default=10), + image = dict(), state = dict(default='present'), zone = dict(default='us-central1-b'), service_account_email = dict(), @@ -147,6 +154,7 @@ def main(): mode = module.params.get('mode') name = module.params.get('name') size_gb = module.params.get('size_gb') + image = module.params.get('image') state = module.params.get('state') zone = module.params.get('zone') @@ -204,8 +212,11 @@ def main(): instance_name, zone), changed=False) if not disk: + lc_image = None + if image is not None: + lc_image = gce.ex_get_image(image) try: - disk = gce.create_volume(size_gb, name, location=zone) + disk = gce.create_volume(size_gb, name, location=zone, image=lc_image) except ResourceExistsError: pass except QuotaExceededError: @@ -214,6 +225,7 @@ def main(): except Exception, e: module.fail_json(msg=unexpected_error_msg(e), changed=False) json_output['size_gb'] = size_gb + json_output['image'] = image changed = True if inst and not is_attached: try: diff --git a/test/integration/roles/test_gce_pd/tasks/main.yml b/test/integration/roles/test_gce_pd/tasks/main.yml index e7c7f15fbe..1a756e1c48 100644 --- a/test/integration/roles/test_gce_pd/tasks/main.yml +++ b/test/integration/roles/test_gce_pd/tasks/main.yml @@ -124,3 +124,40 @@ - 'result.changed' - 'result.name == "{{ instance_name }}"' - 'result.state == "absent"' + +# ============================================================ +- name: test image given (state=present) + gce_pd: + name: "{{ instance_name }}" + image: debian-7 + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert image given (state=present) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.image == "debian-7"' + - 'result.state == "present"' + +# ============================================================ +- name: test image given (state=absent) + gce_pd: + name: "{{ instance_name }}" + image: debian-7 + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert image given (state=absent) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' From d227330a55b87b2479252e7cabf615d14c0f3b91 Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Fri, 11 Apr 2014 15:45:56 -0700 Subject: [PATCH 4/7] Adds support for creating GCE persistent disks from snapshots --- library/cloud/gce_pd | 30 ++++++++-- test/integration/Makefile | 6 +- test/integration/cleanup_gce.py | 6 ++ test/integration/gce_credentials.py | 51 +++++++++++++++++ .../roles/test_gce_pd/tasks/main.yml | 57 +++++++++++++++++++ test/integration/setup_gce.py | 40 +++++++++++++ 6 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 test/integration/gce_credentials.py create mode 100644 test/integration/setup_gce.py diff --git a/library/cloud/gce_pd b/library/cloud/gce_pd index 98e8e8d15e..0032acd57e 100644 --- a/library/cloud/gce_pd +++ b/library/cloud/gce_pd @@ -24,9 +24,7 @@ short_description: utilize GCE persistent disk resources description: - This module can create and destroy unformatted GCE persistent disks U(https://developers.google.com/compute/docs/disks#persistentdisks). - It also supports attaching and detaching disks from running instances - but does not support creating boot disks from images or snapshots. The - 'gce' module supports creating instances with boot disks. + It also supports attaching and detaching disks from running instances. Full install/configuration instructions for the gce* modules can be found in the comments of ansible/test/gce_tests.py. options: @@ -68,6 +66,12 @@ options: required: false default: null aliases: [] + snapshot: + description: + - the source snapshot to use for the disk + required: false + default: null + aliases: [] state: description: - desired state of the persistent disk @@ -139,6 +143,7 @@ def main(): name = dict(required=True), size_gb = dict(default=10), image = dict(), + snapshot = dict(), state = dict(default='present'), zone = dict(default='us-central1-b'), service_account_email = dict(), @@ -155,6 +160,7 @@ def main(): name = module.params.get('name') size_gb = module.params.get('size_gb') image = module.params.get('image') + snapshot = module.params.get('snapshot') state = module.params.get('state') zone = module.params.get('zone') @@ -212,11 +218,20 @@ def main(): instance_name, zone), changed=False) if not disk: + if image is not None and snapshot is not None: + module.fail_json( + msg='Cannot give both image (%s) and snapshot (%s)' % ( + image, snapshot), changed=False) lc_image = None + lc_snapshot = None if image is not None: - lc_image = gce.ex_get_image(image) + lc_image = gce.ex_get_image(image) + elif snapshot is not None: + lc_snapshot = gce.ex_get_snapshot(snapshot) try: - disk = gce.create_volume(size_gb, name, location=zone, image=lc_image) + disk = gce.create_volume( + size_gb, name, location=zone, image=lc_image, + snapshot=lc_snapshot) except ResourceExistsError: pass except QuotaExceededError: @@ -225,7 +240,10 @@ def main(): except Exception, e: module.fail_json(msg=unexpected_error_msg(e), changed=False) json_output['size_gb'] = size_gb - json_output['image'] = image + if image is not None: + json_output['image'] = image + if snapshot is not None: + json_output['snapshot'] = snapshot changed = True if inst and not is_attached: try: diff --git a/test/integration/Makefile b/test/integration/Makefile index d8ca0a10c2..0b0a02003a 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -39,6 +39,9 @@ cloud_cleanup: amazon_cleanup rackspace_cleanup amazon_cleanup: python cleanup_ec2.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" +gce_setup: + python setup_gce.py "$(CLOUD_RESOURCE_PREFIX)" + gce_cleanup: python cleanup_gce.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" @@ -57,7 +60,8 @@ amazon: $(CREDENTIALS_FILE) exit $$RC; gce: $(CREDENTIALS_FILE) - ansible-playbook gce.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -e "resource_prefix=$(CLOUD_RESOURCE_PREFIX)" -v $(TEST_FLAGS) ; \ + CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make gce_setup ; \ + ansible-playbook gce.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -e "resource_prefix=$(CLOUD_RESOURCE_PREFIX)" -v $(TEST_FLAGS) ; \ RC=$$? ; \ CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make gce_cleanup ; \ exit $$RC; diff --git a/test/integration/cleanup_gce.py b/test/integration/cleanup_gce.py index 58b74ca324..70fc65b65d 100644 --- a/test/integration/cleanup_gce.py +++ b/test/integration/cleanup_gce.py @@ -98,6 +98,12 @@ if __name__ == '__main__': try: # Delete matching instances delete_gce_resources(gce.list_nodes, 'name', opts) + # Delete matching snapshots + def get_snapshots(): + for volume in gce.list_volumes(): + for snapshot in gce.list_volume_snapshots(volume): + yield snapshot + delete_gce_resources(get_snapshots, 'name', opts) # Delete matching disks delete_gce_resources(gce.list_volumes, 'name', opts) except KeyboardInterrupt, e: diff --git a/test/integration/gce_credentials.py b/test/integration/gce_credentials.py new file mode 100644 index 0000000000..0d7ae81cae --- /dev/null +++ b/test/integration/gce_credentials.py @@ -0,0 +1,51 @@ +import collections +import os +import yaml + +try: + from libcloud.compute.types import Provider + from libcloud.compute.providers import get_driver + _ = Provider.GCE +except ImportError: + print("failed=True " + \ + "msg='libcloud with GCE support (0.13.3+) required for this module'") + sys.exit(1) + + +def add_credentials_options(parser): + default_service_account_email=None + default_pem_file=None + default_project_id=None + + # Load details from credentials.yml + if os.path.isfile('credentials.yml'): + credentials = yaml.load(open('credentials.yml', 'r')) + default_service_account_email = credentials['gce_service_account_email'] + default_pem_file = credentials['gce_pem_file'] + default_project_id = credentials['gce_project_id'] + + parser.add_option("--service_account_email", + action="store", dest="service_account_email", + default=default_service_account_email, + help="GCE service account email. Default is loaded from credentials.yml.") + parser.add_option("--pem_file", + action="store", dest="pem_file", + default=default_pem_file, + help="GCE client key. Default is loaded from credentials.yml.") + parser.add_option("--project_id", + action="store", dest="project_id", + default=default_project_id, + help="Google Cloud project ID. Default is loaded from credentials.yml.") + + +def check_required(opts, parser): + for required in ['service_account_email', 'pem_file', 'project_id']: + if getattr(opts, required) is None: + parser.error("Missing required parameter: --%s" % required) + + +def get_gce_driver(opts): + # Connect to GCE + gce_cls = get_driver(Provider.GCE) + return gce_cls( + opts.service_account_email, opts.pem_file, project=opts.project_id) diff --git a/test/integration/roles/test_gce_pd/tasks/main.yml b/test/integration/roles/test_gce_pd/tasks/main.yml index 1a756e1c48..70ee1ec10b 100644 --- a/test/integration/roles/test_gce_pd/tasks/main.yml +++ b/test/integration/roles/test_gce_pd/tasks/main.yml @@ -161,3 +161,60 @@ - 'result.changed' - 'result.name == "{{ instance_name }}"' - 'result.state == "absent"' + +# ============================================================ +- name: test snapshot given (state=present) + gce_pd: + name: "{{ instance_name }}" + snapshot: "{{ instance_name }}-snapshot" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert image given (state=present) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.snapshot == "{{ instance_name }}-snapshot"' + - 'result.state == "present"' + +# ============================================================ +- name: test snapshot given (state=absent) + gce_pd: + name: "{{ instance_name }}" + snapshot: "{{ instance_name }}-snapshot" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert image given (state=absent) + assert: + that: + - 'result.changed' + - 'result.name == "{{ instance_name }}"' + - 'result.state == "absent"' + +# ============================================================ +- name: test both image and snapshot given + gce_pd: + name: "{{ instance_name }}" + image: "debian-7" + snapshot: "{{ instance_name }}-snapshot" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + ignore_errors: true + +- name: assert image given (state=present) + assert: + that: + - 'result.failed' + - 'result.msg == "Cannot give both image (debian-7) and snapshot ({{ instance_name }}-snapshot)"' + diff --git a/test/integration/setup_gce.py b/test/integration/setup_gce.py new file mode 100644 index 0000000000..4020b59979 --- /dev/null +++ b/test/integration/setup_gce.py @@ -0,0 +1,40 @@ +''' +Create GCE resources for use in integration tests. + +Takes a prefix as a command-line argument and creates a persistent disk +named ${prefix}-base and a snapshot of it named ${prefix}-snapshot. +prefix will be forced to lowercase, to ensure the names are legal GCE +resource names. +''' + +import sys +import optparse + +import gce_credentials + + +def parse_args(): + parser = optparse.OptionParser( + usage="%s [options] " % (sys.argv[0],), description=__doc__) + gce_credentials.add_credentials_options(parser) + parser.add_option("--prefix", + action="store", dest="prefix", + help="String used to prefix GCE resource names (default: %default)") + + (opts, args) = parser.parse_args() + gce_credentials.check_required(opts, parser) + if not args: + parser.error("Missing required argument: name prefix") + return (opts, args) + +if __name__ == '__main__': + + (opts, args) = parse_args() + gce = gce_credentials.get_gce_driver(opts) + prefix = args[0].lower() + try: + base_volume = gce.create_volume( + size=10, name=prefix+'-base', location='us-central1-a') + gce.create_volume_snapshot(base_volume, name=prefix+'-snapshot') + except KeyboardInterrupt, e: + print "\nExiting on user command." From 2257a6972335bb7e4f8ac2f145c29c39be9d9ad2 Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Sat, 12 Apr 2014 15:21:26 -0700 Subject: [PATCH 5/7] Adds support for attaching persistent disks to GCE instances --- library/cloud/gce | 52 +++++++- .../integration/roles/test_gce/tasks/main.yml | 114 ++++++++++++++++++ test/integration/setup_gce.py | 10 +- 3 files changed, 171 insertions(+), 5 deletions(-) diff --git a/library/cloud/gce b/library/cloud/gce index 2d95c8143b..3484990526 100755 --- a/library/cloud/gce +++ b/library/cloud/gce @@ -89,6 +89,12 @@ options: required: false default: "false" aliases: [] + disks: + description: + - a list of persistent disks to attach to the instance; a string value gives the name of the disk; alternatively, a dictionary value can define 'name' and 'mode' ('READ_ONLY' or 'READ_WRITE'). The first entry will be the boot disk (which must be READ_WRITE). + required: false + default: null + aliases: [] state: description: - desired state of the resource @@ -210,8 +216,16 @@ def get_instance_info(inst): netname = inst.extra['networkInterfaces'][0]['network'].split('/')[-1] except: netname = None + if 'disks' in inst.extra: + disk_names = [disk_info['source'].split('/')[-1] + for disk_info + in sorted(inst.extra['disks'], + key=lambda disk_info: disk_info['index'])] + else: + disk_names = [] return({ 'image': not inst.image is None and inst.image.split('/')[-1] or None, + 'disks': disk_names, 'machine_type': inst.size, 'metadata': metadata, 'name': inst.name, @@ -241,6 +255,7 @@ def create_instances(module, gce, instance_names): metadata = module.params.get('metadata') network = module.params.get('network') persistent_boot_disk = module.params.get('persistent_boot_disk') + disks = module.params.get('disks') state = module.params.get('state') tags = module.params.get('tags') zone = module.params.get('zone') @@ -249,6 +264,16 @@ def create_instances(module, gce, instance_names): changed = False lc_image = gce.ex_get_image(image) + lc_disks = [] + disk_modes = [] + for i, disk in enumerate(disks or []): + if isinstance(disk, dict): + lc_disks.append(gce.ex_get_volume(disk['name'])) + disk_modes.append(disk['mode']) + else: + lc_disks.append(gce.ex_get_volume(disk)) + # boot disk is implicitly READ_WRITE + disk_modes.append('READ_ONLY' if i > 0 else 'READ_WRITE') lc_network = gce.ex_get_network(network) lc_machine_type = gce.ex_get_size(machine_type) lc_zone = gce.ex_get_zone(zone) @@ -283,7 +308,9 @@ def create_instances(module, gce, instance_names): for name in instance_names: pd = None - if persistent_boot_disk: + if lc_disks: + pd = lc_disks[0] + elif persistent_boot_disk: try: pd = gce.create_volume(None, "%s" % name, image=lc_image) except ResourceExistsError: @@ -300,6 +327,28 @@ def create_instances(module, gce, instance_names): module.fail_json(msg='Unexpected error attempting to create ' + \ 'instance %s, error: %s' % (name, e.value)) + for i, lc_disk in enumerate(lc_disks): + # Check whether the disk is already attached + if (len(inst.extra['disks']) > i): + attached_disk = inst.extra['disks'][i] + if attached_disk['source'] != lc_disk.extra['selfLink']: + module.fail_json( + msg=("Disk at index %d does not match: requested=%s found=%s" % ( + i, lc_disk.extra['selfLink'], attached_disk['source']))) + elif attached_disk['mode'] != disk_modes[i]: + module.fail_json( + msg=("Disk at index %d is in the wrong mode: requested=%s found=%s" % ( + i, disk_modes[i], attached_disk['mode']))) + else: + continue + gce.attach_volume(inst, lc_disk, ex_mode=disk_modes[i]) + # Work around libcloud bug: attached volumes don't get added + # to the instance metadata. get_instance_info() only cares about + # source and index. + if len(inst.extra['disks']) != i+1: + inst.extra['disks'].append( + {'source': lc_disk.extra['selfLink'], 'index': i}) + if inst: new_instances.append(inst) @@ -352,6 +401,7 @@ def main(): name = dict(), network = dict(default='default'), persistent_boot_disk = dict(type='bool', default=False), + disks = dict(type='list'), state = dict(choices=['active', 'present', 'absent', 'deleted'], default='present'), tags = dict(type='list'), diff --git a/test/integration/roles/test_gce/tasks/main.yml b/test/integration/roles/test_gce/tasks/main.yml index 8791c281a5..3308dede8b 100644 --- a/test/integration/roles/test_gce/tasks/main.yml +++ b/test/integration/roles/test_gce/tasks/main.yml @@ -95,3 +95,117 @@ - 'result.name == "{{ instance_name }}"' - 'result.state == "absent"' +# ============================================================ +- name: test disks given (expected changed=true) + gce: + name: "{{ instance_name }}" + disks: + - "{{ instance_name }}-base" + - "{{ instance_name }}-extra" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert disks given + assert: + that: + - 'result.changed' + - 'result.instance_data[0].disks == ["{{ instance_name }}-base", "{{ instance_name }}-extra"]' + - 'result.state == "present"' + +# ============================================================ +- name: test disks given (expected changed=false) + gce: + name: "{{ instance_name }}" + disks: + - "{{ instance_name }}-base" + - "{{ instance_name }}-extra" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: present + register: result + +- name: assert disks given + assert: + that: + - 'not result.changed' + - 'result.instance_data[0].disks == ["{{ instance_name }}-base", "{{ instance_name }}-extra"]' + - 'result.state == "present"' + +# ============================================================ +- name: test disks in the wrong order + gce: + name: "{{ instance_name }}" + disks: + - "{{ instance_name }}-extra" + - "{{ instance_name }}-base" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + register: result + ignore_errors: true + +- name: assert disks in the wrong order + assert: + that: + - 'result.failed' + - '{{ result.msg | match("Disk at index 0 does not match:.*") }}' + +# ============================================================ +- name: test disks given with name and mode + gce: + name: "{{ instance_name }}" + disks: + - { name: "{{ instance_name }}-base", mode: "READ_WRITE" } + - { name: "{{ instance_name }}-extra", mode: "READ_ONLY" } + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + register: result + +- name: assert disks given + assert: + that: + - 'not result.changed' + - 'result.state == "present"' + +# ============================================================ +- name: test disks given with name and wrong mode + gce: + name: "{{ instance_name }}" + disks: + - { name: "{{ instance_name }}-base", mode: "READ_ONLY" } + - "{{ instance_name }}-extra" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + register: result + ignore_errors: true + +- name: assert disks given + assert: + that: + - 'result.failed' + - '{{ result.msg | match("Disk at index 0 is in the wrong mode:.*") }}' + +# ============================================================ +- name: test disks given, state absent (expected changed=true) + gce: + name: "{{ instance_name }}" + disks: + - "{{ instance_name }}-base" + - "{{ instance_name }}-extra" + service_account_email: "{{ service_account_email }}" + pem_file: "{{ pem_file }}" + project_id: "{{ project_id }}" + state: absent + register: result + +- name: assert disks given, state absent (expected changed=true) + assert: + that: + - 'result.changed' + - 'result.state == "absent"' diff --git a/test/integration/setup_gce.py b/test/integration/setup_gce.py index 4020b59979..0248d7684d 100644 --- a/test/integration/setup_gce.py +++ b/test/integration/setup_gce.py @@ -1,10 +1,10 @@ ''' Create GCE resources for use in integration tests. -Takes a prefix as a command-line argument and creates a persistent disk -named ${prefix}-base and a snapshot of it named ${prefix}-snapshot. -prefix will be forced to lowercase, to ensure the names are legal GCE -resource names. +Takes a prefix as a command-line argument and creates two persistent disks named +${prefix}-base and ${prefix}-extra and a snapshot of the base disk named +${prefix}-snapshot. prefix will be forced to lowercase, to ensure the names are +legal GCE resource names. ''' import sys @@ -36,5 +36,7 @@ if __name__ == '__main__': base_volume = gce.create_volume( size=10, name=prefix+'-base', location='us-central1-a') gce.create_volume_snapshot(base_volume, name=prefix+'-snapshot') + gce.create_volume( + size=10, name=prefix+'-extra', location='us-central1-a') except KeyboardInterrupt, e: print "\nExiting on user command." From 1e0eed3fcef8d6206fa2d0d841f80e2b1102a8bf Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Sun, 13 Apr 2014 16:16:21 -0700 Subject: [PATCH 6/7] Changes cleanup_gce.py to use gce_credentials module. --- test/integration/cleanup_gce.py | 43 ++++----------------------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/test/integration/cleanup_gce.py b/test/integration/cleanup_gce.py index 70fc65b65d..e0cf0bc043 100644 --- a/test/integration/cleanup_gce.py +++ b/test/integration/cleanup_gce.py @@ -21,6 +21,8 @@ except ImportError: "msg='libcloud with GCE support (0.13.3+) required for this module'") sys.exit(1) +import gce_credentials + def delete_gce_resources(get_func, attr, opts): for item in get_func(): @@ -37,39 +39,9 @@ def prompt_and_delete(item, prompt, assumeyes): print ("Deleted %s" % item) def parse_args(): - default_service_account_email = None - default_pem_file = None - default_project_id = None - - # Load details from credentials.yml - if os.path.isfile('credentials.yml'): - credentials = yaml.load(open('credentials.yml', 'r')) - - if default_service_account_email is None: - default_service_account_email = credentials['gce_service_account_email'] - if default_pem_file is None: - default_pem_file = credentials['gce_pem_file'] - if default_project_id is None: - default_project_id = credentials['gce_project_id'] - parser = optparse.OptionParser(usage="%s [options]" % (sys.argv[0],), description=__doc__) - parser.add_option("--service_account_email", - action="store", dest="service_account_email", - default=default_service_account_email, - help="GCE service account email. Default is loaded from credentials.yml.") - parser.add_option("--pem_file", - action="store", dest="pem_file", - default=default_pem_file, - help="GCE client key. Default is loaded from credentials.yml.") - parser.add_option("--project_id", - action="store", dest="project_id", - default=default_project_id, - help="Google Cloud project ID. Default is loaded from credentials.yml.") - parser.add_option("--credentials", "-c", - action="store", dest="credential_file", - default="credentials.yml", - help="YAML file to read cloud credentials (default: %default)") + gce_credentials.add_credentials_options(parser) parser.add_option("--yes", "-y", action="store_true", dest="assumeyes", default=False, @@ -80,10 +52,7 @@ def parse_args(): help="Regular expression used to find GCE resources (default: %default)") (opts, args) = parser.parse_args() - for required in ['service_account_email', 'pem_file', 'project_id']: - if getattr(opts, required) is None: - parser.error("Missing required parameter: --%s" % required) - + gce_credentials.check_required(opts, parser) return (opts, args) if __name__ == '__main__': @@ -91,9 +60,7 @@ if __name__ == '__main__': (opts, args) = parse_args() # Connect to GCE - gce_cls = get_driver(Provider.GCE) - gce = gce_cls( - opts.service_account_email, opts.pem_file, project=opts.project_id) + gce = gce_credentials.get_gce_driver(opts) try: # Delete matching instances From b9af8ea1d0e8e83102cfa099de1ace0e4f2aa00b Mon Sep 17 00:00:00 2001 From: Chris Conway Date: Mon, 14 Apr 2014 08:33:45 -0700 Subject: [PATCH 7/7] Adds version_added property to new attributes in gce and gce_pd. --- library/cloud/gce | 1 + library/cloud/gce_pd | 2 ++ 2 files changed, 3 insertions(+) diff --git a/library/cloud/gce b/library/cloud/gce index 3484990526..c2ba922812 100755 --- a/library/cloud/gce +++ b/library/cloud/gce @@ -95,6 +95,7 @@ options: required: false default: null aliases: [] + version_added: "1.6" state: description: - desired state of the resource diff --git a/library/cloud/gce_pd b/library/cloud/gce_pd index 0032acd57e..8dd42d04fa 100644 --- a/library/cloud/gce_pd +++ b/library/cloud/gce_pd @@ -66,12 +66,14 @@ options: required: false default: null aliases: [] + version_added: "1.6" snapshot: description: - the source snapshot to use for the disk required: false default: null aliases: [] + version_added: "1.6" state: description: - desired state of the persistent disk