Refactor consul_session to support authentication with tokens (#6755)

* Split into separate PR

* Refactor test, add author to inactive maintainers

* Add changelog fragment and correct requirements section on module documentation

* Add changelog fragment and correct requirements section on module documentation

* Update changelogs/fragments/6755-refactor-consul-session-to-use-requests-lib-instead-of-consul.yml

Co-authored-by: Felix Fontein <felix@fontein.de>

---------

Co-authored-by: Valerio Poggi <vrpoggigmail.com>
Co-authored-by: Felix Fontein <felix@fontein.de>

.github/BOTMETA.yml

@@ -431,7 +431,7 @@ files:
     ignore: resmo
     maintainers: dmtrs
   $modules/consul:
-    ignore: colin-nolan
+    ignore: colin-nolan Hakon
     maintainers: $team_consul
   $modules/copr.py:
     maintainers: schlupov

changelogs/fragments/6755-refactor-consul-session-to-use-requests-lib-instead-of-consul.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - consul_session - drops requirement for the ``python-consul`` library to communicate with the Consul API, instead relying on the existing ``requests`` library requirement (https://github.com/ansible-collections/community.general/pull/6755).

plugins/modules/consul_session.py

@@ -17,10 +17,10 @@ description:
    to implement distributed locks. In depth documentation for working with
    sessions can be found at http://www.consul.io/docs/internals/sessions.html
 requirements:
-  - python-consul
   - requests
 author:
   - Steve Gargan (@sgargan)
+  - Håkon Lerring (@Hakon)
 extends_documentation_fragment:
   - community.general.attributes
 attributes:
@@ -147,15 +147,15 @@ EXAMPLES = '''
     ttl: 600  # sec
 '''
 
-try:
-    import consul
-    from requests.exceptions import ConnectionError
-    python_consul_installed = True
-except ImportError:
-    python_consul_installed = False
-
 from ansible.module_utils.basic import AnsibleModule
 
+try:
+    import requests
+    from requests.exceptions import ConnectionError
+    has_requests = True
+except ImportError:
+    has_requests = False
+
 
 def execute(module):
 
@@ -169,15 +169,74 @@ def execute(module):
         remove_session(module)
 
 
+class RequestError(Exception):
+    pass
+
+
+def handle_consul_response_error(response):
+    if 400 <= response.status_code < 600:
+        raise RequestError('%d %s' % (response.status_code, response.content))
+
+
+def get_consul_url(module):
+    return '%s://%s:%s/v1' % (module.params.get('scheme'),
+                              module.params.get('host'), module.params.get('port'))
+
+
+def get_auth_headers(module):
+    if 'token' in module.params and module.params.get('token') is not None:
+        return {'X-Consul-Token': module.params.get('token')}
+    else:
+        return {}
+
+
+def list_sessions(module, datacenter):
+    url = '%s/session/list' % get_consul_url(module)
+    headers = get_auth_headers(module)
+    response = requests.get(
+        url,
+        headers=headers,
+        params={
+            'dc': datacenter},
+        verify=module.params.get('validate_certs'))
+    handle_consul_response_error(response)
+    return response.json()
+
+
+def list_sessions_for_node(module, node, datacenter):
+    url = '%s/session/node/%s' % (get_consul_url(module), node)
+    headers = get_auth_headers(module)
+    response = requests.get(
+        url,
+        headers=headers,
+        params={
+            'dc': datacenter},
+        verify=module.params.get('validate_certs'))
+    handle_consul_response_error(response)
+    return response.json()
+
+
+def get_session_info(module, session_id, datacenter):
+    url = '%s/session/info/%s' % (get_consul_url(module), session_id)
+    headers = get_auth_headers(module)
+    response = requests.get(
+        url,
+        headers=headers,
+        params={
+            'dc': datacenter},
+        verify=module.params.get('validate_certs'))
+    handle_consul_response_error(response)
+    return response.json()
+
+
 def lookup_sessions(module):
 
     datacenter = module.params.get('datacenter')
 
     state = module.params.get('state')
-    consul_client = get_consul_api(module)
     try:
         if state == 'list':
-            sessions_list = consul_client.session.list(dc=datacenter)
+            sessions_list = list_sessions(module, datacenter)
             # Ditch the index, this can be grabbed from the results
             if sessions_list and len(sessions_list) >= 2:
                 sessions_list = sessions_list[1]
@@ -185,14 +244,14 @@ def lookup_sessions(module):
                              sessions=sessions_list)
         elif state == 'node':
             node = module.params.get('node')
-            sessions = consul_client.session.node(node, dc=datacenter)
+            sessions = list_sessions_for_node(module, node, datacenter)
             module.exit_json(changed=True,
                              node=node,
                              sessions=sessions)
         elif state == 'info':
             session_id = module.params.get('id')
 
-            session_by_id = consul_client.session.info(session_id, dc=datacenter)
+            session_by_id = get_session_info(module, session_id, datacenter)
             module.exit_json(changed=True,
                              session_id=session_id,
                              sessions=session_by_id)
@@ -201,6 +260,31 @@ def lookup_sessions(module):
         module.fail_json(msg="Could not retrieve session info %s" % e)
 
 
+def create_session(module, name, behavior, ttl, node,
+                   lock_delay, datacenter, checks):
+    url = '%s/session/create' % get_consul_url(module)
+    headers = get_auth_headers(module)
+    create_data = {
+        "LockDelay": lock_delay,
+        "Node": node,
+        "Name": name,
+        "Checks": checks,
+        "Behavior": behavior,
+    }
+    if ttl is not None:
+        create_data["TTL"] = "%ss" % str(ttl)  # TTL is in seconds
+    response = requests.put(
+        url,
+        headers=headers,
+        params={
+            'dc': datacenter},
+        json=create_data,
+        verify=module.params.get('validate_certs'))
+    handle_consul_response_error(response)
+    create_session_response_dict = response.json()
+    return create_session_response_dict["ID"]
+
+
 def update_session(module):
 
     name = module.params.get('name')
@@ -211,16 +295,14 @@ def update_session(module):
     behavior = module.params.get('behavior')
     ttl = module.params.get('ttl')
 
-    consul_client = get_consul_api(module)
-
     try:
-        session = consul_client.session.create(
+        session = create_session(module,
                                  name=name,
                                  behavior=behavior,
                                  ttl=ttl,
                                  node=node,
                                  lock_delay=delay,
-            dc=datacenter,
+                                 datacenter=datacenter,
                                  checks=checks
                                  )
         module.exit_json(changed=True,
@@ -235,13 +317,22 @@ def update_session(module):
         module.fail_json(msg="Could not create/update session %s" % e)
 
 
+def destroy_session(module, session_id):
+    url = '%s/session/destroy/%s' % (get_consul_url(module), session_id)
+    headers = get_auth_headers(module)
+    response = requests.put(
+        url,
+        headers=headers,
+        verify=module.params.get('validate_certs'))
+    handle_consul_response_error(response)
+    return response.content == "true"
+
+
 def remove_session(module):
     session_id = module.params.get('id')
 
-    consul_client = get_consul_api(module)
-
     try:
-        consul_client.session.destroy(session_id)
+        destroy_session(module, session_id)
 
         module.exit_json(changed=True,
                          session_id=session_id)
@@ -250,25 +341,22 @@ def remove_session(module):
                          session_id, e))
 
 
-def get_consul_api(module):
-    return consul.Consul(host=module.params.get('host'),
-                         port=module.params.get('port'),
-                         scheme=module.params.get('scheme'),
-                         verify=module.params.get('validate_certs'),
-                         token=module.params.get('token'))
-
-
 def test_dependencies(module):
-    if not python_consul_installed:
+    if not has_requests:
-        module.fail_json(msg="python-consul required for this module. "
+        raise ImportError(
-                             "see https://python-consul.readthedocs.io/en/latest/#installation")
+            "requests required for this module. See https://pypi.org/project/requests/")
 
 
 def main():
     argument_spec = dict(
         checks=dict(type='list', elements='str'),
         delay=dict(type='int', default='15'),
-        behavior=dict(type='str', default='release', choices=['release', 'delete']),
+        behavior=dict(
+            type='str',
+            default='release',
+            choices=[
+                'release',
+                'delete']),
         ttl=dict(type='int'),
         host=dict(type='str', default='localhost'),
         port=dict(type='int', default=8500),
@@ -277,7 +365,15 @@ def main():
         id=dict(type='str'),
         name=dict(type='str'),
         node=dict(type='str'),
-        state=dict(type='str', default='present', choices=['absent', 'info', 'list', 'node', 'present']),
+        state=dict(
+            type='str',
+            default='present',
+            choices=[
+                'absent',
+                'info',
+                'list',
+                'node',
+                'present']),
         datacenter=dict(type='str'),
         token=dict(type='str', no_log=True),
     )

tests/integration/targets/consul/tasks/consul_session.yml

@@ -6,6 +6,7 @@
 - name: list sessions
   consul_session:
     state: list
+    token: "{{ consul_management_token }}"
   register: result
 
 - assert:
@@ -17,6 +18,7 @@
   consul_session:
     state: present
     name: testsession
+    token: "{{ consul_management_token }}"
   register: result
 
 - assert:
@@ -31,6 +33,7 @@
 - name: list sessions after creation
   consul_session:
     state: list
+    token: "{{ consul_management_token }}"
   register: result
 
 - set_fact:
@@ -52,12 +55,13 @@
 - name: ensure session was created
   assert:
     that:
-      - test_session_found|default(False)
+      - test_session_found|default(false)
 
 - name: fetch info about a session
   consul_session:
     state: info
     id: '{{ session_id }}'
+    token: "{{ consul_management_token }}"
   register: result
 
 - assert:
@@ -68,6 +72,7 @@
   consul_session:
     state: info
     name: test
+    token: "{{ consul_management_token }}"
   register: result
   ignore_errors: true
 
@@ -80,6 +85,7 @@
     state: info
     id: '{{ session_id }}'
     scheme: non_existent
+    token: "{{ consul_management_token }}"
   register: result
   ignore_errors: true
 
@@ -93,6 +99,7 @@
     id: '{{ session_id }}'
     port: 8501
     scheme: https
+    token: "{{ consul_management_token }}"
   register: result
   ignore_errors: true
 
@@ -108,6 +115,7 @@
     id: '{{ session_id }}'
     port: 8501
     scheme: https
+    token: "{{ consul_management_token }}"
     validate_certs: false
   register: result
 
@@ -122,6 +130,7 @@
     id: '{{ session_id }}'
     port: 8501
     scheme: https
+    token: "{{ consul_management_token }}"
   environment:
     REQUESTS_CA_BUNDLE: '{{ remote_dir }}/cert.pem'
   register: result
@@ -134,6 +143,7 @@
   consul_session:
     state: absent
     id: '{{ session_id }}'
+    token: "{{ consul_management_token }}"
   register: result
 
 - assert:
@@ -143,6 +153,7 @@
 - name: list sessions after deletion
   consul_session:
     state: list
+    token: "{{ consul_management_token }}"
   register: result
 
 - assert:
@@ -169,6 +180,7 @@
     state: present
     name: session-with-ttl
     ttl: 180  # sec
+    token: "{{ consul_management_token }}"
   register: result
 
 - assert:

tests/integration/targets/consul/tasks/main.yml

@@ -10,8 +10,8 @@
 
 - name: Install Consul and test
   vars:
-    consul_version: 1.5.0
+    consul_version: 1.13.2
-    consul_uri: https://s3.amazonaws.com/ansible-ci-files/test/integration/targets/consul/consul_{{ consul_version }}_{{ ansible_system | lower }}_{{ consul_arch }}.zip
+    consul_uri: https://releases.hashicorp.com/consul/{{ consul_version }}/consul_{{ consul_version }}_{{ ansible_system | lower }}_{{ consul_arch }}.zip
     consul_cmd: '{{ remote_tmp_dir }}/consul'
   block:
   - name: Install requests<2.20 (CentOS/RHEL 6)
@@ -76,8 +76,15 @@
         dest: '{{ remote_tmp_dir }}/consul_config.hcl'
     - name: Start Consul (dev mode enabled)
       shell: nohup {{ consul_cmd }} agent -dev -config-file {{ remote_tmp_dir }}/consul_config.hcl </dev/null >/dev/null 2>&1 &
+    - name: Bootstrap ACL
+      command: '{{ consul_cmd }} acl bootstrap --format=json'
+      register: consul_bootstrap_result_string
+    - set_fact:
+        consul_management_token: '{{ consul_bootstrap_json_result["SecretID"] }}'
+      vars:
+        consul_bootstrap_json_result: '{{ consul_bootstrap_result_string.stdout | from_json }}'
     - name: Create some data
-      command: '{{ consul_cmd }} kv put data/value{{ item }} foo{{ item }}'
+      command: '{{ consul_cmd }} kv put -token={{consul_management_token}} data/value{{ item }} foo{{ item }}'
       loop:
       - 1
       - 2

tests/integration/targets/consul/templates/consul_config.hcl.j2

@@ -12,3 +12,8 @@ ports {
 }
 key_file = "{{ remote_dir }}/privatekey.pem"
 cert_file = "{{ remote_dir }}/cert.pem"
+acl {
+  enabled        = true
+  default_policy = "deny"
+  down_policy    = "extend-cache"
+}