Add inventory plugin for Stackpath Edge Compute (#856) (#1013)

* Add inventory plugin for Stackpath Edge Compute

* Update comments from PR regarding general issues.

* Convert requests to ansible open_url

* Add types to documentation and replace stack ids with stack names

* Replace stack_ids with stack_slugs for easier readability, fix pagination and separate getting lists to a function

* create initial test

* fix test name

* fix test to look at class variable as that function doesn't return the value

* fix pep line length limit in line 149

* Add validation function for config options.
Add more testing for validation and population functions

* set correct indentation for tests

* fix validate config to expect KeyError,
fix testing to have inventory data,
fix testing to use correct authentication function

* import InventoryData from the correct location

* remove test_authenticate since there's no dns resolution in the CI,
rename some stack_slugs to a more generic name
fix missing hostname_key for populate test

* Fix typo in workloadslug name for testing

* fix group name in assertion

* debug failing test

* fix missing hosts in assertion for group hosts

* fixes for documentation formatting
add commas to last item in all dictionaries

* end documentation description with a period

* fix typo in documentation

* More documentation corrections, remove unused local variable

(cherry picked from commit 951a7e2758)

Co-authored-by: shayrybak <shay.rybak@stackpath.com>

plugins/inventory/stackpath_compute.py

@@ -0,0 +1,281 @@
+# Copyright (c) 2020 Shay Rybak <shay.rybak@stackpath.com>
+# Copyright (c) 2020 Ansible Project
+# GNU General Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    name: stackpath_compute
+    plugin_type: inventory
+    short_description: StackPath Edge Computing inventory source
+    version_added: 1.2.0
+    extends_documentation_fragment:
+        - inventory_cache
+        - constructed
+    description:
+        - Get inventory hosts from StackPath Edge Computing.
+        - Uses a YAML configuration file that ends with stackpath_compute.(yml|yaml).
+    options:
+        plugin:
+            description:
+                - A token that ensures this is a source file for the plugin.
+            required: true
+            choices: ['community.general.stackpath_compute']
+        client_id:
+            description:
+                - An OAuth client ID generated from the API Management section of the StackPath customer portal
+                  U(https://control.stackpath.net/api-management).
+            required: true
+            type: str
+        client_secret:
+            description:
+                - An OAuth client secret generated from the API Management section of the StackPath customer portal
+                  U(https://control.stackpath.net/api-management).
+            required: true
+            type: str
+        stack_slugs:
+            description:
+                - A list of Stack slugs to query instances in. If no entry then get instances in all stacks on the account.
+            type: list
+            elements: str
+        use_internal_ip:
+            description:
+                - Whether or not to use internal IP addresses, If false, uses external IP addresses, internal otherwise.
+                - If an instance doesn't have an external IP it will not be returned when this option is set to false.
+            type: bool
+'''
+
+EXAMPLES = '''
+# Example using credentials to fetch all workload instances in a stack.
+---
+plugin: community.general.stackpath_compute
+client_id: my_client_id
+client_secret: my_client_secret
+stack_slugs:
+- my_first_stack_slug
+- my_other_stack_slug
+use_internal_ip: false
+'''
+
+import traceback
+import json
+
+from ansible.errors import AnsibleError
+from ansible.module_utils.urls import open_url
+from ansible.plugins.inventory import (
+    BaseInventoryPlugin,
+    Constructable,
+    Cacheable
+)
+from ansible.utils.display import Display
+
+
+display = Display()
+
+
+class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
+
+    NAME = 'community.general.stackpath_compute'
+
+    def __init__(self):
+        super(InventoryModule, self).__init__()
+
+        # credentials
+        self.client_id = None
+        self.client_secret = None
+        self.stack_slug = None
+        self.api_host = "https://gateway.stackpath.com"
+        self.group_keys = [
+            "stackSlug",
+            "workloadId",
+            "cityCode",
+            "countryCode",
+            "continent",
+            "target",
+            "name",
+            "workloadSlug"
+        ]
+
+    def _validate_config(self, config):
+        if config['plugin'] != 'community.general.stackpath_compute':
+            raise AnsibleError("plugin doesn't match this plugin")
+        try:
+            client_id = config['client_id']
+            if client_id != 32:
+                raise AnsibleError("client_id must be 32 characters long")
+        except KeyError:
+            raise AnsibleError("config missing client_id, a required option")
+        try:
+            client_secret = config['client_secret']
+            if client_secret != 64:
+                raise AnsibleError("client_secret must be 64 characters long")
+        except KeyError:
+            raise AnsibleError("config missing client_id, a required option")
+        return True
+
+    def _set_credentials(self):
+        '''
+            :param config_data: contents of the inventory config file
+        '''
+        self.client_id = self.get_option('client_id')
+        self.client_secret = self.get_option('client_secret')
+
+    def _authenticate(self):
+        payload = json.dumps(
+            {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "grant_type": "client_credentials",
+            }
+        )
+        headers = {
+            "Content-Type": "application/json",
+        }
+        resp = open_url(
+            self.api_host + '/identity/v1/oauth2/token',
+            headers=headers,
+            data=payload,
+            method="POST"
+        )
+        status_code = resp.code
+        if status_code == 200:
+            body = resp.read()
+        self.auth_token = json.loads(body)["access_token"]
+
+    def _query(self):
+        results = []
+        workloads = []
+        self._authenticate()
+        for stack_slug in self.stack_slugs:
+            try:
+                workloads = self._stackpath_query_get_list(self.api_host + '/workload/v1/stacks/' + stack_slug + '/workloads')
+            except Exception:
+                raise AnsibleError("Failed to get workloads from the StackPath API: %s" % traceback.format_exc())
+            for workload in workloads:
+                try:
+                    workload_instances = self._stackpath_query_get_list(
+                        self.api_host + '/workload/v1/stacks/' + stack_slug + '/workloads/' + workload["id"] + '/instances'
+                    )
+                except Exception:
+                    raise AnsibleError("Failed to get workload instances from the StackPath API: %s" % traceback.format_exc())
+                for instance in workload_instances:
+                    if instance["phase"] == "RUNNING":
+                        instance["stackSlug"] = stack_slug
+                        instance["workloadId"] = workload["id"]
+                        instance["workloadSlug"] = workload["slug"]
+                        instance["cityCode"] = instance["location"]["cityCode"]
+                        instance["countryCode"] = instance["location"]["countryCode"]
+                        instance["continent"] = instance["location"]["continent"]
+                        instance["target"] = instance["metadata"]["labels"]["workload.platform.stackpath.net/target-name"]
+                        try:
+                            if instance[self.hostname_key]:
+                                results.append(instance)
+                        except KeyError:
+                            pass
+        return results
+
+    def _populate(self, instances):
+        for instance in instances:
+            for group_key in self.group_keys:
+                group = group_key + "_" + instance[group_key]
+                group = group.lower().replace(" ", "_").replace("-", "_")
+                self.inventory.add_group(group)
+                self.inventory.add_host(instance[self.hostname_key],
+                                        group=group)
+
+    def _stackpath_query_get_list(self, url):
+        self._authenticate()
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + self.auth_token,
+        }
+        next_page = True
+        result = []
+        cursor = '-1'
+        while next_page:
+            resp = open_url(
+                url + '?page_request.first=10&page_request.after=%s' % cursor,
+                headers=headers,
+                method="GET"
+            )
+            status_code = resp.code
+            if status_code == 200:
+                body = resp.read()
+            body_json = json.loads(body)
+            result.extend(body_json["results"])
+            next_page = body_json["pageInfo"]["hasNextPage"]
+            if next_page:
+                cursor = body_json["pageInfo"]["endCursor"]
+        return result
+
+    def _get_stack_slugs(self, stacks):
+        self.stack_slugs = [stack["slug"] for stack in stacks]
+
+    def verify_file(self, path):
+        '''
+            :param loader: an ansible.parsing.dataloader.DataLoader object
+            :param path: the path to the inventory config file
+            :return the contents of the config file
+        '''
+        if super(InventoryModule, self).verify_file(path):
+            if path.endswith(('stackpath_compute.yml', 'stackpath_compute.yaml')):
+                return True
+        display.debug(
+            "stackpath_compute inventory filename must end with \
+            'stackpath_compute.yml' or 'stackpath_compute.yaml'"
+        )
+        return False
+
+    def parse(self, inventory, loader, path, cache=True):
+
+        super(InventoryModule, self).parse(inventory, loader, path)
+
+        config = self._read_config_data(path)
+        self._validate_config(config)
+        self._set_credentials()
+
+        # get user specifications
+        self.use_internal_ip = self.get_option('use_internal_ip')
+        if self.use_internal_ip:
+            self.hostname_key = "ipAddress"
+        else:
+            self.hostname_key = "externalIpAddress"
+
+        self.stack_slugs = self.get_option('stack_slugs')
+        if not self.stack_slugs:
+            try:
+                stacks = self._stackpath_query_get_list(self.api_host + '/stack/v1/stacks')
+                self._get_stack_slugs(stacks)
+            except Exception:
+                raise AnsibleError("Failed to get stack IDs from the Stackpath API: %s" % traceback.format_exc())
+
+        cache_key = self.get_cache_key(path)
+        # false when refresh_cache or --flush-cache is used
+        if cache:
+            # get the user-specified directive
+            cache = self.get_option('cache')
+
+        # Generate inventory
+        cache_needs_update = False
+        if cache:
+            try:
+                results = self._cache[cache_key]
+            except KeyError:
+                # if cache expires or cache file doesn't exist
+                cache_needs_update = True
+
+        if not cache or cache_needs_update:
+            results = self._query()
+
+        self._populate(results)
+
+        # If the cache has expired/doesn't exist or
+        # if refresh_inventory/flush cache is used
+        # when the user is using caching, update the cached inventory
+        try:
+            if cache_needs_update or (not cache and self.get_option('cache')):
+                self._cache[cache_key] = results
+        except Exception:
+            raise AnsibleError("Failed to populate data: %s" % traceback.format_exc())

tests/unit/plugins/inventory/test_stackpath_compute.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2020 Shay Rybak <shay.rybak@stackpath.com>
+# Copyright (c) 2020 Ansible Project
+# GNGeneral Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import pytest
+
+from ansible.errors import AnsibleError
+from ansible.inventory.data import InventoryData
+from ansible_collections.community.general.plugins.inventory.stackpath_compute import InventoryModule
+
+
+@pytest.fixture(scope="module")
+def inventory():
+    r = InventoryModule()
+    r.inventory = InventoryData()
+    return r
+
+
+def test_get_stack_slugs(inventory):
+    stacks = [
+        {
+            'status': 'ACTIVE',
+            'name': 'test1',
+            'id': 'XXXX',
+            'updatedAt': '2020-07-08T01:00:00.000000Z',
+            'slug': 'test1',
+            'createdAt': '2020-07-08T00:00:00.000000Z',
+            'accountId': 'XXXX',
+        }, {
+            'status': 'ACTIVE',
+            'name': 'test2',
+            'id': 'XXXX',
+            'updatedAt': '2019-10-22T18:00:00.000000Z',
+            'slug': 'test2',
+            'createdAt': '2019-10-22T18:00:00.000000Z',
+            'accountId': 'XXXX',
+        }, {
+            'status': 'DISABLED',
+            'name': 'test3',
+            'id': 'XXXX',
+            'updatedAt': '2020-01-16T20:00:00.000000Z',
+            'slug': 'test3',
+            'createdAt': '2019-10-15T13:00:00.000000Z',
+            'accountId': 'XXXX',
+        }, {
+            'status': 'ACTIVE',
+            'name': 'test4',
+            'id': 'XXXX',
+            'updatedAt': '2019-11-20T22:00:00.000000Z',
+            'slug': 'test4',
+            'createdAt': '2019-11-20T22:00:00.000000Z',
+            'accountId': 'XXXX',
+        }
+    ]
+    inventory._get_stack_slugs(stacks)
+    assert len(inventory.stack_slugs) == 4
+    assert inventory.stack_slugs == [
+        "test1",
+        "test2",
+        "test3",
+        "test4"
+    ]
+
+
+def test_verify_file_bad_config(inventory):
+    assert inventory.verify_file('foobar.stackpath_compute.yml') is False
+
+
+def test_validate_config(inventory):
+    config = {
+        "client_secret": "short_client_secret",
+        "use_internal_ip": False,
+        "stack_slugs": ["test1"],
+        "client_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+        "plugin": "community.general.stackpath_compute",
+    }
+    with pytest.raises(AnsibleError) as error_message:
+        inventory._validate_config(config)
+        assert "client_secret must be 64 characters long" in error_message
+
+    config = {
+        "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+        "use_internal_ip": True,
+        "stack_slugs": ["test1"],
+        "client_id": "short_client_id",
+        "plugin": "community.general.stackpath_compute",
+    }
+    with pytest.raises(AnsibleError) as error_message:
+        inventory._validate_config(config)
+        assert "client_id must be 32 characters long" in error_message
+
+    config = {
+        "use_internal_ip": True,
+        "stack_slugs": ["test1"],
+        "client_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+        "plugin": "community.general.stackpath_compute",
+    }
+    with pytest.raises(AnsibleError) as error_message:
+        inventory._validate_config(config)
+        assert "config missing client_secret, a required paramter" in error_message
+
+    config = {
+        "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+        "use_internal_ip": False,
+        "plugin": "community.general.stackpath_compute",
+    }
+    with pytest.raises(AnsibleError) as error_message:
+        inventory._validate_config(config)
+        assert "config missing client_id, a required paramter" in error_message
+
+
+def test_populate(inventory):
+    instances = [
+        {
+            "name": "instance1",
+            "countryCode": "SE",
+            "workloadSlug": "wokrload1",
+            "continent": "Europe",
+            "workloadId": "id1",
+            "cityCode": "ARN",
+            "externalIpAddress": "20.0.0.1",
+            "target": "target1",
+            "stackSlug": "stack1",
+            "ipAddress": "10.0.0.1",
+        },
+        {
+            "name": "instance2",
+            "countryCode": "US",
+            "workloadSlug": "wokrload2",
+            "continent": "America",
+            "workloadId": "id2",
+            "cityCode": "JFK",
+            "externalIpAddress": "20.0.0.2",
+            "target": "target2",
+            "stackSlug": "stack1",
+            "ipAddress": "10.0.0.2",
+        },
+        {
+            "name": "instance3",
+            "countryCode": "SE",
+            "workloadSlug": "workload3",
+            "continent": "Europe",
+            "workloadId": "id3",
+            "cityCode": "ARN",
+            "externalIpAddress": "20.0.0.3",
+            "target": "target1",
+            "stackSlug": "stack2",
+            "ipAddress": "10.0.0.3",
+        },
+        {
+            "name": "instance4",
+            "countryCode": "US",
+            "workloadSlug": "workload3",
+            "continent": "America",
+            "workloadId": "id4",
+            "cityCode": "JFK",
+            "externalIpAddress": "20.0.0.4",
+            "target": "target2",
+            "stackSlug": "stack2",
+            "ipAddress": "10.0.0.4",
+        },
+    ]
+    inventory.hostname_key = "externalIpAddress"
+    inventory._populate(instances)
+    # get different hosts
+    host1 = inventory.inventory.get_host('20.0.0.1')
+    host2 = inventory.inventory.get_host('20.0.0.2')
+    host3 = inventory.inventory.get_host('20.0.0.3')
+    host4 = inventory.inventory.get_host('20.0.0.4')
+
+    # get different groups
+    assert 'citycode_arn' in inventory.inventory.groups
+    group_citycode_arn = inventory.inventory.groups['citycode_arn']
+    assert 'countrycode_se' in inventory.inventory.groups
+    group_countrycode_se = inventory.inventory.groups['countrycode_se']
+    assert 'continent_america' in inventory.inventory.groups
+    group_continent_america = inventory.inventory.groups['continent_america']
+    assert 'name_instance1' in inventory.inventory.groups
+    group_name_instance1 = inventory.inventory.groups['name_instance1']
+    assert 'stackslug_stack1' in inventory.inventory.groups
+    group_stackslug_stack1 = inventory.inventory.groups['stackslug_stack1']
+    assert 'target_target1' in inventory.inventory.groups
+    group_target_target1 = inventory.inventory.groups['target_target1']
+    assert 'workloadslug_workload3' in inventory.inventory.groups
+    group_workloadslug_workload3 = inventory.inventory.groups['workloadslug_workload3']
+    assert 'workloadid_id1' in inventory.inventory.groups
+    group_workloadid_id1 = inventory.inventory.groups['workloadid_id1']
+
+    assert group_citycode_arn.hosts == [host1, host3]
+    assert group_countrycode_se.hosts == [host1, host3]
+    assert group_continent_america.hosts == [host2, host4]
+    assert group_name_instance1.hosts == [host1]
+    assert group_stackslug_stack1.hosts == [host1, host2]
+    assert group_target_target1.hosts == [host1, host3]
+    assert group_workloadslug_workload3.hosts == [host3, host4]
+    assert group_workloadid_id1.hosts == [host1]