From 9c967dd05492ab837914fd588f7d9cbe22654ec4 Mon Sep 17 00:00:00 2001 From: willthames Date: Thu, 13 Feb 2014 12:52:38 +1000 Subject: [PATCH 1/6] Allow ec2 inventory to use a boto profile This allows the EC2 inventory plugin to be used with the same configuration against different EC2 accounts Profile can be passed using --profile variable or using EC2_PROFILE environment variable e.g. ``` EC2_PROFILE=prod ansible-playbook -i ec2.py playbook.yml ``` Added documentation on profiles to EC2 dynamic inventory doc Only tries to use profiles if --profile argument is given or EC2_PROFILE is set to maintain compatibility will boto < 2.24. Works around a minor bug in boto where if you try and use a security token with a profile it fails (boto/boto#2100) --- contrib/inventory/ec2.py | 51 +++++++++++++++++++++++-- docsite/rst/intro_dynamic_inventory.rst | 14 +++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index 7ed9b83e77..6af748f918 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -22,6 +22,12 @@ you need to define: export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus +If you're using boto profiles (requires boto>=2.24.0) you can choose a profile +using the --profile command line argument (e.g. ec2.py --profile prod) or using +the EC2_PROFILE variable: + + EC2_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml + For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html When run against a specific host, this script returns the following variables: @@ -148,9 +154,20 @@ class Ec2Inventory(object): # Index of hostname (address) to instance ID self.index = {} - # Read settings and parse CLI arguments - self.read_settings() + # Parse CLI arguments and read settings self.parse_cli_args() + self.read_settings() + + # boto profile to use (if any) + # Make sure that profile_name is not passed at all if not set + # as pre 2.24 boto will fall over otherwise + if self.args.profile: + if not hasattr(boto.ec2.EC2Connection, 'profile_name'): + sys.stderr.write("boto version must be >= 2.24 to use profile\n") + sys.exit(1) + self.profile = dict(profile_name=self.args.profile) + else: + self.profile = dict() # Cache if self.args.refresh_cache: @@ -292,6 +309,8 @@ class Ec2Inventory(object): # Cache related cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) + if self.args.profile: + cache_dir = os.path.join(cache_dir, 'profile_' + self.args.profile) if not os.path.exists(cache_dir): os.makedirs(cache_dir) @@ -373,6 +392,8 @@ class Ec2Inventory(object): help='Get all the variables about a specific instance') parser.add_argument('--refresh-cache', action='store_true', default=False, help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') + parser.add_argument('--profile', action='store', default=os.environ.get('EC2_PROFILE'), + help='Use boto profile for connections to EC2') self.args = parser.parse_args() @@ -405,6 +426,21 @@ class Ec2Inventory(object): self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) return conn + def boto_fix_security_token_in_profile(self, conn): + ''' monkey patch for boto issue boto/boto#2100 ''' + profile = 'profile ' + self.profile.get('profile_name') + if boto.config.has_option(profile, 'aws_security_token'): + conn.provider.set_security_token(boto.config.get(profile, 'aws_security_token')) + return conn + + + def connect_to_aws(self, module, region): + conn = module.connect_to_region(region, **self.profile) + if 'profile_name' in self.profile: + conn = self.boto_fix_security_token_in_profile(conn) + return conn + + def get_instances_by_region(self, region): ''' Makes an AWS EC2 API call to the list of instances in a particular region ''' @@ -416,8 +452,14 @@ class Ec2Inventory(object): for filter_key, filter_values in self.ec2_instance_filters.items(): reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) else: - reservations = conn.get_all_instances() + conn = self.connect_to_aws(ec2, region) + # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported + if conn is None: + print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) + sys.exit(1) + + reservations = conn.get_all_instances() for reservation in reservations: for instance in reservation.instances: self.add_instance(instance, region) @@ -430,12 +472,13 @@ class Ec2Inventory(object): error = "Error connecting to %s backend.\n%s" % (backend, e.message) self.fail_with_error(error, 'getting EC2 instances') + def get_rds_instances_by_region(self, region): ''' Makes an AWS API call to the list of RDS instances in a particular region ''' try: - conn = rds.connect_to_region(region) + conn = self.connect_to_aws(rds, region) if conn: instances = conn.get_all_dbinstances() for instance in instances: diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index 5b634d86cd..78ed6809f1 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -101,6 +101,20 @@ You can test the script by itself to make sure your config is correct:: After a few moments, you should see your entire EC2 inventory across all regions in JSON. +If you use boto profiles to manage multiple AWS accounts, you can pass ``--profile PROFILE`` name to the ``ec2.py`` script. An example profile might be:: + + [profile dev] + aws_access_key_id = + aws_secret_access_key = + + [profile prod] + aws_access_key_id = + aws_secret_access_key = + +You can then run ``ec2.py --profile prod`` to get the inventory for the prod account, or run playbooks with: ``ansible-playbook -i 'ec2.py --profile prod' myplaybook.yml``. + +Alternatively, use the ``EC2_PROFILE`` variable - e.g. ``EC2_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml`` + Since each region requires its own API call, if you are only using a small set of regions, feel free to edit ``ec2.ini`` and list only the regions you are interested in. There are other config options in ``ec2.ini`` including cache control, and destination variables. At their heart, inventory files are simply a mapping from some name to a destination address. The default ``ec2.ini`` settings are configured for running Ansible from outside EC2 (from your laptop for example) -- and this is not the most efficient way to manage EC2. From 8c11ea5666168ce5e9cd1f42e9cc486af5f40da5 Mon Sep 17 00:00:00 2001 From: Jeff '2 bits' Bachtel Date: Tue, 12 Aug 2014 16:01:36 -0400 Subject: [PATCH 2/6] Add an ec2 inventory option "boto_profile" that allows the use of boto profiles for separating credentials as specified in http://boto.readthedocs.org/en/latest/boto_config_tut.html --- contrib/inventory/ec2.ini | 4 ++++ contrib/inventory/ec2.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/contrib/inventory/ec2.ini b/contrib/inventory/ec2.ini index 50430ce0ed..93be4c8acb 100644 --- a/contrib/inventory/ec2.ini +++ b/contrib/inventory/ec2.ini @@ -139,3 +139,7 @@ group_by_elasticache_replication_group = True # tag Name value matches webservers1* # (ex. webservers15, webservers1a, webservers123 etc) # instance_filters = tag:Name=webservers1* + +# A boto configuration profile may be used to separate out credentials +# see http://boto.readthedocs.org/en/latest/boto_config_tut.html +# boto_profile = some-boto-profile-name diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index 6af748f918..d34291f256 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -318,6 +318,11 @@ class Ec2Inventory(object): self.cache_path_index = cache_dir + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age') + # boto configuration profile + self.boto_profile = None + if config.has_option('ec2', 'boto_profile'): + self.boto_profile = config.get('ec2', 'boto_profile') + # Configure nested groups instead of flat namespace. if config.has_option('ec2', 'nested_groups'): self.nested_groups = config.getboolean('ec2', 'nested_groups') @@ -452,7 +457,7 @@ class Ec2Inventory(object): for filter_key, filter_values in self.ec2_instance_filters.items(): reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) else: - conn = self.connect_to_aws(ec2, region) + conn = ec2.connect_to_region(region, profile_name=self.boto_profile) # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported if conn is None: From e7890e66f3ec9224f01871ee0e7268e581c74ae6 Mon Sep 17 00:00:00 2001 From: Jameel Al-Aziz Date: Wed, 24 Sep 2014 15:28:42 -0700 Subject: [PATCH 3/6] Combine ansible/ansible#5987 and ansible/ansible#8582 Fixes merge conflicts and standardizes option naming --- contrib/inventory/ec2.py | 55 ++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index d34291f256..7b0c7efeb5 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -22,11 +22,11 @@ you need to define: export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus -If you're using boto profiles (requires boto>=2.24.0) you can choose a profile -using the --profile command line argument (e.g. ec2.py --profile prod) or using -the EC2_PROFILE variable: +If you're using boto profiles (requires boto>=2.24.0) you can choose a profile +using the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using +the AWS_PROFILE variable: - EC2_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml + AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html @@ -154,20 +154,19 @@ class Ec2Inventory(object): # Index of hostname (address) to instance ID self.index = {} - # Parse CLI arguments and read settings + # Boto profile to use (if any) + self.boto_profile = None + + # Read settings and parse CLI arguments self.parse_cli_args() self.read_settings() - # boto profile to use (if any) # Make sure that profile_name is not passed at all if not set # as pre 2.24 boto will fall over otherwise - if self.args.profile: + if self.boto_profile: if not hasattr(boto.ec2.EC2Connection, 'profile_name'): sys.stderr.write("boto version must be >= 2.24 to use profile\n") sys.exit(1) - self.profile = dict(profile_name=self.args.profile) - else: - self.profile = dict() # Cache if self.args.refresh_cache: @@ -307,10 +306,15 @@ class Ec2Inventory(object): else: self.all_elasticache_nodes = False + # boto configuration profile (prefer CLI argument) + self.boto_profile = self.args.boto_profile + if config.has_option('ec2', 'boto_profile') and not self.boto_profile: + self.boto_profile = config.get('ec2', 'boto_profile') + # Cache related cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) - if self.args.profile: - cache_dir = os.path.join(cache_dir, 'profile_' + self.args.profile) + if self.boto_profile: + cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile) if not os.path.exists(cache_dir): os.makedirs(cache_dir) @@ -318,11 +322,6 @@ class Ec2Inventory(object): self.cache_path_index = cache_dir + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age') - # boto configuration profile - self.boto_profile = None - if config.has_option('ec2', 'boto_profile'): - self.boto_profile = config.get('ec2', 'boto_profile') - # Configure nested groups instead of flat namespace. if config.has_option('ec2', 'nested_groups'): self.nested_groups = config.getboolean('ec2', 'nested_groups') @@ -387,6 +386,7 @@ class Ec2Inventory(object): continue self.ec2_instance_filters[filter_key].append(filter_value) + def parse_cli_args(self): ''' Command line argument processing ''' @@ -397,7 +397,7 @@ class Ec2Inventory(object): help='Get all the variables about a specific instance') parser.add_argument('--refresh-cache', action='store_true', default=False, help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') - parser.add_argument('--profile', action='store', default=os.environ.get('EC2_PROFILE'), + parser.add_argument('--boto-profile', action='store', help='Use boto profile for connections to EC2') self.args = parser.parse_args() @@ -431,18 +431,23 @@ class Ec2Inventory(object): self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) return conn - def boto_fix_security_token_in_profile(self, conn): + def boto_fix_security_token_in_profile(self, connect_args): ''' monkey patch for boto issue boto/boto#2100 ''' - profile = 'profile ' + self.profile.get('profile_name') + profile = 'profile ' + self.boto_profile if boto.config.has_option(profile, 'aws_security_token'): - conn.provider.set_security_token(boto.config.get(profile, 'aws_security_token')) - return conn + connect_args['secuirty_token'] = boto.config.get(profile, 'aws_security_token') + return connect_args def connect_to_aws(self, module, region): - conn = module.connect_to_region(region, **self.profile) - if 'profile_name' in self.profile: - conn = self.boto_fix_security_token_in_profile(conn) + connect_args = {} + + # only pass the profile name if it's set (as it is not supported by older boto versions) + if self.boto_profile: + connect_args['profile_name'] = self.boto_profile + self.boto_fix_security_token_in_profile(connect_args) + + conn = module.connect_to_region(region, **connect_args) return conn From 57754b5a71a1fc2061b1117cf17dbb1d57507b3a Mon Sep 17 00:00:00 2001 From: Jameel Al-Aziz Date: Fri, 10 Oct 2014 15:33:26 -0700 Subject: [PATCH 4/6] Fix security_token typo in ec2 inventory --- contrib/inventory/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index 7b0c7efeb5..98a5df0601 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -435,7 +435,7 @@ class Ec2Inventory(object): ''' monkey patch for boto issue boto/boto#2100 ''' profile = 'profile ' + self.boto_profile if boto.config.has_option(profile, 'aws_security_token'): - connect_args['secuirty_token'] = boto.config.get(profile, 'aws_security_token') + connect_args['security_token'] = boto.config.get(profile, 'aws_security_token') return connect_args From c08137a3d2ced0ed6cc44e5a6e72e795ccfcb16a Mon Sep 17 00:00:00 2001 From: Jameel Al-Aziz Date: Mon, 27 Oct 2014 23:03:53 -0700 Subject: [PATCH 5/6] Update ec2 inventory documentation to refer to AWS_PROFILE --- docsite/rst/intro_dynamic_inventory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index 78ed6809f1..1a2bd6f72c 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -113,7 +113,7 @@ If you use boto profiles to manage multiple AWS accounts, you can pass ``--profi You can then run ``ec2.py --profile prod`` to get the inventory for the prod account, or run playbooks with: ``ansible-playbook -i 'ec2.py --profile prod' myplaybook.yml``. -Alternatively, use the ``EC2_PROFILE`` variable - e.g. ``EC2_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml`` +Alternatively, use the ``AWS_PROFILE`` variable - e.g. ``AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml`` Since each region requires its own API call, if you are only using a small set of regions, feel free to edit ``ec2.ini`` and list only the regions you are interested in. There are other config options in ``ec2.ini`` including cache control, and destination variables. From b0133d9c8f9554cce94e069140e1dba67542a352 Mon Sep 17 00:00:00 2001 From: Will Thames Date: Mon, 27 Jul 2015 11:44:05 +1000 Subject: [PATCH 6/6] Move connection handling failure to connect_to_aws Make use of better error handling mechanism --- contrib/inventory/ec2.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index 98a5df0601..6fc57f6c95 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -165,8 +165,7 @@ class Ec2Inventory(object): # as pre 2.24 boto will fall over otherwise if self.boto_profile: if not hasattr(boto.ec2.EC2Connection, 'profile_name'): - sys.stderr.write("boto version must be >= 2.24 to use profile\n") - sys.exit(1) + self.fail_with_error("boto version must be >= 2.24 to use profile") # Cache if self.args.refresh_cache: @@ -386,7 +385,6 @@ class Ec2Inventory(object): continue self.ec2_instance_filters[filter_key].append(filter_value) - def parse_cli_args(self): ''' Command line argument processing ''' @@ -425,10 +423,7 @@ class Ec2Inventory(object): conn = boto.connect_euca(host=self.eucalyptus_host) conn.APIVersion = '2010-08-31' else: - conn = ec2.connect_to_region(region) - # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported - if conn is None: - self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) + conn = self.connect_to_aws(ec2, region) return conn def boto_fix_security_token_in_profile(self, connect_args): @@ -438,7 +433,6 @@ class Ec2Inventory(object): connect_args['security_token'] = boto.config.get(profile, 'aws_security_token') return connect_args - def connect_to_aws(self, module, region): connect_args = {} @@ -448,9 +442,11 @@ class Ec2Inventory(object): self.boto_fix_security_token_in_profile(connect_args) conn = module.connect_to_region(region, **connect_args) + # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported + if conn is None: + self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) return conn - def get_instances_by_region(self, region): ''' Makes an AWS EC2 API call to the list of instances in a particular region ''' @@ -462,14 +458,8 @@ class Ec2Inventory(object): for filter_key, filter_values in self.ec2_instance_filters.items(): reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) else: - conn = ec2.connect_to_region(region, profile_name=self.boto_profile) + reservations = conn.get_all_instances() - # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported - if conn is None: - print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) - sys.exit(1) - - reservations = conn.get_all_instances() for reservation in reservations: for instance in reservation.instances: self.add_instance(instance, region) @@ -482,7 +472,6 @@ class Ec2Inventory(object): error = "Error connecting to %s backend.\n%s" % (backend, e.message) self.fail_with_error(error, 'getting EC2 instances') - def get_rds_instances_by_region(self, region): ''' Makes an AWS API call to the list of RDS instances in a particular region '''