From 40f9da351f171aca102d93b7924fe914c1c88a70 Mon Sep 17 00:00:00 2001 From: Joseph Tate Date: Fri, 13 Dec 2013 13:43:30 -0500 Subject: [PATCH 1/3] Extend ec2 module to support spot instances --- library/cloud/ec2 | 107 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 25 deletions(-) diff --git a/library/cloud/ec2 b/library/cloud/ec2 index 0e0b8aaf0f..3442605da8 100644 --- a/library/cloud/ec2 +++ b/library/cloud/ec2 @@ -67,6 +67,12 @@ options: required: true default: null aliases: [] + spot_price: + description: + - Maximum spot price to bid, If not set a regular on-demand instance is requested. A spot request is made with this maximum bid. When it is filled, the instance is started. + required: false + default: null + aliases: [] image: description: - I(emi) (or I(ami)) to use for the instance @@ -97,6 +103,11 @@ options: - how long before wait gives up, in seconds default: 300 aliases: [] + spot_wait_timeout: + description: + - how long to wait for the spot instance request to be fulfilled + default: 600 + aliases: [] ec2_url: description: - Url to use to connect to EC2 or your Eucalyptus cloud (by default the module will use EC2 endpoints). Must be specified if region is not used. If not set then the value of the EC2_URL environment variable, if any, is used @@ -247,6 +258,19 @@ local_action: vpc_subnet_id: subnet-29e63245 assign_public_ip: yes +# Spot instance example +- local_action: + module: ec2 + spot_price: 0.24 + spot_wait_timeout: 600 + keypair: mykey + group_id: sg-1dc53f72 + instance_type: m1.small + image: ami-6e649707 + wait: yes + vpc_subnet_id: subnet-29e63245 + assign_public_ip: yes + # Launch instances, runs some tasks # and then terminate them @@ -392,6 +416,7 @@ def create_instances(module, ec2): group_id = module.params.get('group_id') zone = module.params.get('zone') instance_type = module.params.get('instance_type') + spot_price = module.params.get('spot_price') image = module.params.get('image') count = module.params.get('count') monitoring = module.params.get('monitoring') @@ -399,6 +424,7 @@ def create_instances(module, ec2): ramdisk = module.params.get('ramdisk') wait = module.params.get('wait') wait_timeout = int(module.params.get('wait_timeout')) + spot_wait_timeout = int(module.params.get('spot_wait_timeout')) placement_group = module.params.get('placement_group') user_data = module.params.get('user_data') instance_tags = module.params.get('instance_tags') @@ -456,16 +482,12 @@ def create_instances(module, ec2): try: params = {'image_id': image, 'key_name': key_name, - 'client_token': id, - 'min_count': count_remaining, - 'max_count': count_remaining, 'monitoring_enabled': monitoring, 'placement': zone, 'placement_group': placement_group, 'instance_type': instance_type, 'kernel_id': kernel, 'ramdisk_id': ramdisk, - 'private_ip_address': private_ip, 'user_data': user_data} if boto_supports_profile_name_arg(ec2): @@ -498,22 +520,55 @@ def create_instances(module, ec2): else: params['security_groups'] = group_name - res = ec2.run_instances(**params) + if not spot_price: + params.update({ + 'min_count': count_remaining, + 'max_count': count_remaining, + 'client_token': id, + 'private_ip_address': private_ip, + }) + res = ec2.run_instances(**params) + instids = [ i.id for i in res.instances ] + while True: + try: + ec2.get_all_instances(instids) + break + except boto.exception.EC2ResponseError as e: + if "InvalidInstanceID.NotFound" in str(e): + # there's a race between start and get an instance + continue + else: + module.fail_json(msg = str(e)) + else: + if private_ip: + module.fail_json( + msg='private_ip only available with on-demand (non-spot) instances') + params.update({ + 'count': count_remaining, + }) + res = ec2.request_spot_instances(spot_price, **params) + #Now we have to do the intermediate waiting + if wait: + spot_req_inst_ids = dict() + spot_wait_timeout = time.time() + spot_wait_timeout + while spot_wait_timeout > time.time(): + reqs = ec2.get_all_spot_instance_requests() + for sirb in res: + if sirb.id in spot_req_inst_ids: + continue + for sir in reqs: + if sir.id == sirb.id and sir.instance_id is not None: + spot_req_inst_ids[sirb.id] = sir.instance_id + if len(spot_req_inst_ids) < count: + time.sleep(5) + else: + break + if spot_wait_timeout <= time.time(): + module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime()) + instids = spot_req_inst_ids.values() except boto.exception.BotoServerError, e: module.fail_json(msg = "%s: %s" % (e.error_code, e.error_message)) - instids = [ i.id for i in res.instances ] - while True: - try: - res.connection.get_all_instances(instids) - break - except boto.exception.EC2ResponseError as e: - if "InvalidInstanceID.NotFound" in str(e): - # there's a race between start and get an instance - continue - else: - module.fail_json(msg = str(e)) - if instance_tags: try: ec2.create_tags(instids, instance_tags) @@ -521,15 +576,14 @@ def create_instances(module, ec2): module.fail_json(msg = "%s: %s" % (e.error_code, e.error_message)) # wait here until the instances are up - this_res = [] num_running = 0 wait_timeout = time.time() + wait_timeout while wait_timeout > time.time() and num_running < len(instids): - res_list = res.connection.get_all_instances(instids) - if len(res_list) > 0: - this_res = res_list[0] - num_running = len([ i for i in this_res.instances if i.state=='running' ]) - else: + res_list = ec2.get_all_instances(instids) + num_running = 0 + for res in res_list: + num_running += len([ i for i in res.instances if i.state=='running' ]) + if len(res_list) <= 0: # got a bad response of some sort, possibly due to # stale/cached data. Wait a second and then try again time.sleep(1) @@ -543,8 +597,9 @@ def create_instances(module, ec2): # waiting took too long module.fail_json(msg = "wait for instances running timeout on %s" % time.asctime()) - for inst in this_res.instances: - running_instances.append(inst) + #We do this after the loop ends so that we end up with one list + for res in res_list: + running_instances.extend(res.instances) instance_dict_array = [] created_instance_ids = [] @@ -631,6 +686,7 @@ def main(): region = dict(aliases=['aws_region', 'ec2_region'], choices=AWS_REGIONS), zone = dict(aliases=['aws_zone', 'ec2_zone']), instance_type = dict(aliases=['type']), + spot_price = dict(), image = dict(), kernel = dict(), count = dict(default='1'), @@ -638,6 +694,7 @@ def main(): ramdisk = dict(), wait = dict(type='bool', default=False), wait_timeout = dict(default=300), + spot_wait_timeout = dict(default=600), ec2_url = dict(), ec2_secret_key = dict(aliases=['aws_secret_key', 'secret_key'], no_log=True), ec2_access_key = dict(aliases=['aws_access_key', 'access_key']), From 080e70ab6e39940e1ee5bdd39f600aa586814667 Mon Sep 17 00:00:00 2001 From: Joseph Tate Date: Fri, 13 Dec 2013 15:01:58 -0500 Subject: [PATCH 2/3] Added version_added for spot instance parameters --- library/cloud/ec2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/cloud/ec2 b/library/cloud/ec2 index 3442605da8..dac64c3243 100644 --- a/library/cloud/ec2 +++ b/library/cloud/ec2 @@ -68,6 +68,7 @@ options: default: null aliases: [] spot_price: + version_added: "1.5" description: - Maximum spot price to bid, If not set a regular on-demand instance is requested. A spot request is made with this maximum bid. When it is filled, the instance is started. required: false @@ -104,6 +105,7 @@ options: default: 300 aliases: [] spot_wait_timeout: + version_added: "1.5" description: - how long to wait for the spot instance request to be fulfilled default: 600 From e868d0047233e0fd7d36756bf02a707a69b2bcc7 Mon Sep 17 00:00:00 2001 From: Joseph Tate Date: Thu, 19 Dec 2013 18:16:56 -0500 Subject: [PATCH 3/3] Add capability check for parameters on request_spot_instances --- library/cloud/ec2 | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/library/cloud/ec2 b/library/cloud/ec2 index dac64c3243..c992122e7c 100644 --- a/library/cloud/ec2 +++ b/library/cloud/ec2 @@ -399,6 +399,17 @@ def boto_supports_profile_name_arg(ec2): run_instances_method = getattr(ec2, 'run_instances') return 'instance_profile_name' in run_instances_method.func_code.co_varnames +def boto_supports_param_in_spot_request(ec2, param): + """ + Check if Boto library has a in its request_spot_instances() method. For example, the placement_group parameter wasn't added until 2.3.0. + + ec2: authenticated ec2 connection object + + Returns: + True if boto library has the named param as an argument on the request_spot_instances method, else False + """ + method = getattr(ec2, 'request_spot_instances') + return param in method.func_code.co_varnames def create_instances(module, ec2): """ @@ -486,7 +497,6 @@ def create_instances(module, ec2): 'key_name': key_name, 'monitoring_enabled': monitoring, 'placement': zone, - 'placement_group': placement_group, 'instance_type': instance_type, 'kernel_id': kernel, 'ramdisk_id': ramdisk, @@ -527,6 +537,7 @@ def create_instances(module, ec2): 'min_count': count_remaining, 'max_count': count_remaining, 'client_token': id, + 'placement_group': placement_group, 'private_ip_address': private_ip, }) res = ec2.run_instances(**params) @@ -545,6 +556,12 @@ def create_instances(module, ec2): if private_ip: module.fail_json( msg='private_ip only available with on-demand (non-spot) instances') + if boto_supports_param_in_spot_request(ec2, placement_group): + params['placement_group'] = placement_group + elif placement_group : + module.fail_json( + msg="placement_group parameter requires Boto version 2.3.0 or higher.") + params.update({ 'count': count_remaining, })