From f97243d6ed7dc7bb23c9cf675ae84c116beae457 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Thu, 20 Mar 2014 15:50:08 +0100 Subject: [PATCH 1/7] ec2_group: Add support for handling egress rules --- library/cloud/ec2_group | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index 1dd463cc8d..e0b2bc8502 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -135,6 +135,7 @@ def main(): description=dict(required=True), vpc_id=dict(), rules=dict(), + rules_egress=dict(), state = dict(default='present', choices=['present', 'absent']), ) ) @@ -147,6 +148,7 @@ def main(): description = module.params['description'] vpc_id = module.params['vpc_id'] rules = module.params['rules'] + rules_egress = module.params['rules_egress'] state = module.params.get('state') changed = False @@ -203,6 +205,8 @@ def main(): # create a lookup for all existing rules on the group if group: + + # Manage ingress rules groupRules = {} addRulesToLookup(group.rules, 'in', groupRules) @@ -260,6 +264,76 @@ def main(): group.revoke(rule.ip_protocol, rule.from_port, rule.to_port, grant.cidr_ip, grantGroup) changed = True + # Manage egress rules + groupRules = {} + addRulesToLookup(group.rules_egress, 'out', groupRules) + + # Now, go through all provided rules and ensure they are there. + if rules_egress: + for rule in rules_egress: + group_id = None + group_name = None + ip = None + if 'group_id' in rule and 'cidr_ip' in rule: + module.fail_json(msg="Specify group_id OR cidr_ip, not both") + elif 'group_name' in rule and 'cidr_ip' in rule: + module.fail_json(msg="Specify group_name OR cidr_ip, not both") + elif 'group_id' in rule and 'group_name' in rule: + module.fail_json(msg="Specify group_id OR group_name, not both") + elif 'group_id' in rule: + group_id = rule['group_id'] + elif 'group_name' in rule: + group_name = rule['group_name'] + if group_name in groups: + group_id = groups[group_name].id + elif group_name == name: + group_id = group.id + groups[group_id] = group + groups[group_name] = group + elif 'cidr_ip' in rule: + ip = rule['cidr_ip'] + + if rule['proto'] == 'all': + rule['proto'] = -1 + rule['from_port'] = None + rule['to_port'] = None + + # If rule already exists, don't later delete it + ruleId = "%s-%s-%s-%s-%s-%s" % ('out', rule['proto'], rule['from_port'], rule['to_port'], group_id, ip) + if ruleId in groupRules: + del groupRules[ruleId] + # Otherwise, add new rule + else: + grantGroup = None + if group_id: + grantGroup = groups[group_id].id + + if not module.check_mode: + ec2.authorize_security_group_egress( + group_id=group.id, + ip_protocol=rule['proto'], + from_port=rule['from_port'], + to_port=rule['to_port'], + src_group_id=grantGroup, + cidr_ip=ip) + changed = True + + # Finally, remove anything left in the groupRules -- these will be defunct rules + for rule in groupRules.itervalues(): + for grant in rule.grants: + grantGroup = None + if grant.group_id: + grantGroup = groups[grant.group_id].id + if not module.check_mode: + ec2.revoke_security_group_egress( + group_id=group.id, + ip_protocol=rule.ip_protocol, + from_port=rule.from_port, + to_port=rule.to_port, + src_group_id=grantGroup, + cidr_ip=grant.cidr_ip) + changed = True + if group: module.exit_json(changed=changed, group_id=group.id) else: From 77d7165dde0b5f5642ec3667854d55ca2482602c Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Thu, 20 Mar 2014 16:19:35 +0100 Subject: [PATCH 2/7] ec2_group: Deduplicate rule parsing/validation code --- library/cloud/ec2_group | 83 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index e0b2bc8502..d685b29aa0 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -128,6 +128,45 @@ def addRulesToLookup(rules, prefix, dict): dict["%s-%s-%s-%s-%s-%s" % (prefix, rule.ip_protocol, rule.from_port, rule.to_port, grant.group_id, grant.cidr_ip)] = rule + +def get_target_from_rule(rule, name, groups): + """ + Returns tuple of (group_id, ip) after validating rule params. + + rule: Dict describing a rule. + name: Name of the security group being managed. + groups: Dict of all available security groups. + + AWS accepts an ip range or a security group as target of a rule. This + function validate the rule specification and return either a non-None + group_id or a non-None ip range. + """ + + group_id = None + group_name = None + ip = None + if 'group_id' in rule and 'cidr_ip' in rule: + module.fail_json(msg="Specify group_id OR cidr_ip, not both") + elif 'group_name' in rule and 'cidr_ip' in rule: + module.fail_json(msg="Specify group_name OR cidr_ip, not both") + elif 'group_id' in rule and 'group_name' in rule: + module.fail_json(msg="Specify group_id OR group_name, not both") + elif 'group_id' in rule: + group_id = rule['group_id'] + elif 'group_name' in rule: + group_name = rule['group_name'] + if group_name in groups: + group_id = groups[group_name].id + elif group_name == name: + group_id = group.id + groups[group_id] = group + groups[group_name] = group + elif 'cidr_ip' in rule: + ip = rule['cidr_ip'] + + return group_id, ip + + def main(): argument_spec = ec2_argument_spec() argument_spec.update(dict( @@ -213,27 +252,7 @@ def main(): # Now, go through all provided rules and ensure they are there. if rules: for rule in rules: - group_id = None - group_name = None - ip = None - if 'group_id' in rule and 'cidr_ip' in rule: - module.fail_json(msg="Specify group_id OR cidr_ip, not both") - elif 'group_name' in rule and 'cidr_ip' in rule: - module.fail_json(msg="Specify group_name OR cidr_ip, not both") - elif 'group_id' in rule and 'group_name' in rule: - module.fail_json(msg="Specify group_id OR group_name, not both") - elif 'group_id' in rule: - group_id = rule['group_id'] - elif 'group_name' in rule: - group_name = rule['group_name'] - if group_name in groups: - group_id = groups[group_name].id - elif group_name == name: - group_id = group.id - groups[group_id] = group - groups[group_name] = group - elif 'cidr_ip' in rule: - ip = rule['cidr_ip'] + group_id, ip = get_target_from_rule(rule, name, groups) if rule['proto'] == 'all': rule['proto'] = -1 @@ -271,27 +290,7 @@ def main(): # Now, go through all provided rules and ensure they are there. if rules_egress: for rule in rules_egress: - group_id = None - group_name = None - ip = None - if 'group_id' in rule and 'cidr_ip' in rule: - module.fail_json(msg="Specify group_id OR cidr_ip, not both") - elif 'group_name' in rule and 'cidr_ip' in rule: - module.fail_json(msg="Specify group_name OR cidr_ip, not both") - elif 'group_id' in rule and 'group_name' in rule: - module.fail_json(msg="Specify group_id OR group_name, not both") - elif 'group_id' in rule: - group_id = rule['group_id'] - elif 'group_name' in rule: - group_name = rule['group_name'] - if group_name in groups: - group_id = groups[group_name].id - elif group_name == name: - group_id = group.id - groups[group_id] = group - groups[group_name] = group - elif 'cidr_ip' in rule: - ip = rule['cidr_ip'] + group_id, ip = get_target_from_rule(rule, name, groups) if rule['proto'] == 'all': rule['proto'] = -1 From 649fcd3e1cc72c480a1c87c6985a319488f64afa Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Thu, 20 Mar 2014 16:38:54 +0100 Subject: [PATCH 3/7] ec2_group: Auto create missing groups referenced in rules Suppose a pair of groups, A and B, depending on each other. One solution for breaking the circular dependency at playbook level: - declare group A without dependencies - declare group B depending on A - declare group A depending on B This patch breaks the dependency at module level. Whenever a depended-on group is missing it's first created. This approach requires only two tasks: - declare group A depending on B (group B will be auto created) - declare group B depending on A When creating a group EC2 requires you to pass the group description. In order to fullfil this, rules now accept the `group_desc` param. Note that group description can't be changed once the group is created so it's nice to keep descriptions in sync. Concrete example: - ec2_group: name: mysql-client description: MySQL Client rules_egress: - proto: tcp from_port: 3306 to_port: 3306 group_name: mysql-server group_desc: MySQL Server - ec2_group: name: mysql-server description: MySQL Server rules: - proto: tcp from_port: 3306 to_port: 3306 group_name: mysql-client --- library/cloud/ec2_group | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index d685b29aa0..ede8050c0a 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -145,6 +145,7 @@ def get_target_from_rule(rule, name, groups): group_id = None group_name = None ip = None + target_group_created = False if 'group_id' in rule and 'cidr_ip' in rule: module.fail_json(msg="Specify group_id OR cidr_ip, not both") elif 'group_name' in rule and 'cidr_ip' in rule: @@ -161,10 +162,19 @@ def get_target_from_rule(rule, name, groups): group_id = group.id groups[group_id] = group groups[group_name] = group + else: + if not rule.get('group_desc', '').strip(): + module.fail_json(msg="group %s will be automatically created by rule %s and no description was provided" % (group_name, rule)) + if not module.check_mode: + auto_group = ec2.create_security_group(group_name, rule['group_desc'], vpc_id=vpc_id) + group_id = auto_group.id + groups[group_id] = auto_group + groups[group_name] = auto_group + target_group_created = True elif 'cidr_ip' in rule: ip = rule['cidr_ip'] - return group_id, ip + return group_id, ip, target_group_created def main(): @@ -252,7 +262,9 @@ def main(): # Now, go through all provided rules and ensure they are there. if rules: for rule in rules: - group_id, ip = get_target_from_rule(rule, name, groups) + group_id, ip, target_group_created = get_target_from_rule(rule, name, groups) + if target_group_created: + changed = True if rule['proto'] == 'all': rule['proto'] = -1 @@ -290,7 +302,9 @@ def main(): # Now, go through all provided rules and ensure they are there. if rules_egress: for rule in rules_egress: - group_id, ip = get_target_from_rule(rule, name, groups) + group_id, ip, target_group_created = get_target_from_rule(rule, name, groups) + if target_group_created: + changed = True if rule['proto'] == 'all': rule['proto'] = -1 From 3156df9931e3eabb1f3a7dccf02a73e35d97c228 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Thu, 20 Mar 2014 17:20:21 +0100 Subject: [PATCH 4/7] ec2_group: Request a fresh group object after creation When a group is created, an egress_rule ALLOW ALL to 0.0.0.0/0 is added automatically but it's not reflected in the object returned by the AWS API call. After creation we re-read the group for getting an updated object. --- library/cloud/ec2_group | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index ede8050c0a..f15756c97a 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -248,6 +248,12 @@ def main(): '''no match found, create it''' if not module.check_mode: group = ec2.create_security_group(name, description, vpc_id=vpc_id) + + # When a group is created, an egress_rule ALLOW ALL + # to 0.0.0.0/0 is added automatically but it's not + # reflected in the object returned by the AWS API + # call. We re-read the group for getting an updated object + group = ec2.get_all_security_groups(group_ids=(group.id,))[0] changed = True else: module.fail_json(msg="Unsupported state requested: %s" % state) From b642e39e675db2d848c38cd8912410f5ed5bfec7 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Thu, 20 Mar 2014 17:22:56 +0100 Subject: [PATCH 5/7] ec2_group: rules are not a required task argument --- library/cloud/ec2_group | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index f15756c97a..cf290c3474 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -25,7 +25,7 @@ options: rules: description: - List of firewall rules to enforce in this group (see example). - required: true + required: false region: description: - the EC2 region to use From a429ff26dc8a9c9abc663eaeb5810b5e6eab5bec Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Thu, 20 Mar 2014 17:23:53 +0100 Subject: [PATCH 6/7] ec2_group: Add documentation for rules_egress --- library/cloud/ec2_group | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index cf290c3474..e25185c5f1 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -24,7 +24,11 @@ options: required: false rules: description: - - List of firewall rules to enforce in this group (see example). + - List of firewall inbound rules to enforce in this group (see example). + required: false + rules_egress: + description: + - List of firewall outbound rules to enforce in this group (see example). required: false region: description: @@ -113,6 +117,11 @@ EXAMPLES = ''' - proto: all # the containing group name may be specified here group_name: example + rules_egress: + - proto: tcp + from_port: 80 + to_port: 80 + group_name: example ''' try: From 6a92d48b0f053617c42dd4c28e042b971b4259e0 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Fri, 21 Mar 2014 08:35:25 +0100 Subject: [PATCH 7/7] ec2_group: Document group_desc rule param --- library/cloud/ec2_group | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/library/cloud/ec2_group b/library/cloud/ec2_group index e25185c5f1..bf40e7b83b 100644 --- a/library/cloud/ec2_group +++ b/library/cloud/ec2_group @@ -85,6 +85,11 @@ options: version_added: "1.6" requirements: [ "boto" ] + +notes: + - If a rule declares a group_name and that group doesn't exist, it will be + automatically created. In that case, group_desc should be provided as well. + The module will refuse to create a depended-on group without a description. ''' EXAMPLES = ''' @@ -121,7 +126,9 @@ EXAMPLES = ''' - proto: tcp from_port: 80 to_port: 80 - group_name: example + group_name: example-other + # description to use if example-other needs to be created + group_desc: other example EC2 group ''' try: