Fix and rework gitlab_project_variable (#4038)

* rework-and-fix

* fix delete bug and change report

* delete the requested variables based on env scope

* fix absent logic when not purge: remove what is requested

* change code to current behaviour

* complete implementation

* fix delete

* restore origin return structure

* reorder

* add test for origin bug

* add changelog fragment

* Update plugins/modules/source_control/gitlab/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/modules/source_control/gitlab/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/modules/source_control/gitlab/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* remove yaml

* apply suggestions

* readd accidental removed line

* improve the truth of return value 'project_variable' in check mode

* fix pep8, over-indented

* fix typos and add subelement options

* Update changelogs/fragments/4038-fix-and-rework-gitlb-project-variable.yml

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update changelogs/fragments/4038-fix-and-rework-gitlb-project-variable.yml

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/modules/source_control/gitlab/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/modules/source_control/gitlab/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/modules/source_control/gitlab/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* remove diff feature

* resolve all recommentdations

* resolve change requests, improve doc and remove default value before compare, because values always exists (prebuild)

Co-authored-by: Felix Fontein <felix@fontein.de>

changelogs/fragments/4038-fix-and-rework-gitlb-project-variable.yml

@@ -0,0 +1,9 @@
+bugfixes:
+  - >
+    gitlab_project_variable - allow to set same variable name under different environment scopes.
+    Due this change, the return value ``project_variable`` differs from previous version in check mode.
+    It was counting ``updated`` values, because it was accidentally overwriting environment scopes (https://github.com/ansible-collections/community.general/issues/4038).
+  - gitlab_project_variable - fix idempotent change behaviour for float and integer variables (https://github.com/ansible-collections/community.general/issues/4038).
+  - gitlab_project_variable - add missing documentation about GitLab versions that support ``environment_scope`` and ``variable_type`` (https://github.com/ansible-collections/community.general/issues/4038).
+minor_changes:
+  - gitlab_project_variable - new ``variables`` parameter (https://github.com/ansible-collections/community.general/issues/4038).

plugins/modules/source_control/gitlab/gitlab_project_variable.py

@@ -48,6 +48,8 @@ options:
         have full control about whether a value should be masked, protected or both.
       - Support for protected values requires GitLab >= 9.3.
       - Support for masked values requires GitLab >= 11.10.
+      - Support for environment_scope requires GitLab Premium >= 13.11.
+      - Support for variable_type requires GitLab >= 11.11.
       - A I(value) must be a string or a number.
       - Field I(variable_type) must be a string with either C(env_var), which is the default, or C(file).
       - Field I(environment_scope) must be a string defined by scope environment.
@@ -55,6 +57,50 @@ options:
         See GitLab documentation on acceptable values for a masked variable (https://docs.gitlab.com/ce/ci/variables/#masked-variables).
     default: {}
     type: dict
+  variables:
+    version_added: 4.4.0
+    description:
+      - A list of dictionaries that represents CI/CD variables.
+      - This module works internal with this structure, even if the older I(vars) parameter is used.
+    default: []
+    type: list
+    elements: dict
+    suboptions:
+      name:
+        description:
+          - The name of the variable.
+        type: str
+        required: true
+      value:
+        description:
+          - The variable value.
+        type: str
+        required: true
+      masked:
+        description:
+          - Wether variable value is masked or not.
+          - Support for masked values requires GitLab >= 11.10.
+        type: bool
+        default: false
+      protected:
+        description:
+          - Wether variable value is protected or not.
+          - Support for protected values requires GitLab >= 9.3.
+        type: bool
+        default: false
+      variable_type:
+        description:
+          - Wether a variable is an environment variable (C(env_var)) or a file (C(file)).
+          - Support for I(variable_type) requires GitLab >= 11.11.
+        type: str
+        choices: ["env_var", "file"]
+        default: env_var
+      environment_scope:
+        description:
+          - The scope for the variable.
+          - Support for I(environment_scope) requires GitLab Premium >= 13.11.
+        type: str
+        default: '*'
 '''
 
 
@@ -65,9 +111,14 @@ EXAMPLES = '''
     api_token: secret_access_token
     project: markuman/dotfiles
     purge: false
-    vars:
-      ACCESS_KEY_ID: abc123
-      SECRET_ACCESS_KEY: 321cba
+    variables:
+      - name: ACCESS_KEY_ID
+        value: abc123
+      - name: SECRET_ACCESS_KEY
+        value: dassgrfaeui8989
+        masked: yes
+        protected: yes
+        environment_scope: production
 
 - name: Set or update some CI/CD variables
   community.general.gitlab_project_variable:
@@ -123,14 +174,12 @@ project_variable:
 '''
 
 import traceback
-
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.api import basic_auth_argument_spec
 from ansible.module_utils.six import string_types
 from ansible.module_utils.six import integer_types
 
-
 GITLAB_IMP_ERR = None
 try:
     import gitlab
@@ -142,6 +191,43 @@ except Exception:
 from ansible_collections.community.general.plugins.module_utils.gitlab import auth_argument_spec, gitlab_authentication
 
 
+def vars_to_variables(vars, module):
+    # transform old vars to new variables structure
+    variables = list()
+    for item, value in vars.items():
+        if (isinstance(value, string_types) or
+           isinstance(value, (integer_types, float))):
+            variables.append(
+                {
+                    "name": item,
+                    "value": str(value),
+                    "masked": False,
+                    "protected": False,
+                    "variable_type": "env_var",
+                }
+            )
+
+        elif isinstance(value, dict):
+
+            new_item = {
+                "name": item,
+                "value": value.get('value'),
+                "masked": value.get('masked'),
+                "protected": value.get('protected'),
+                "variable_type": value.get('variable_type'),
+            }
+
+            if value.get('environment_scope'):
+                new_item['environment_scope'] = value.get('environment_scope')
+
+            variables.append(new_item)
+
+        else:
+            module.fail_json(msg="value must be of type string, integer, float or dict")
+
+    return variables
+
+
 class GitlabProjectVariables(object):
 
     def __init__(self, module, gitlab_instance):
@@ -162,106 +248,150 @@ class GitlabProjectVariables(object):
             vars_page = self.project.variables.list(page=page_nb)
         return variables
 
-    def create_variable(self, key, value, masked, protected, variable_type, environment_scope):
+    def create_variable(self, var_obj):
         if self._module.check_mode:
-            return
+            return True
+
         var = {
-            "key": key, "value": value,
-            "masked": masked, "protected": protected,
-            "variable_type": variable_type,
+            "key": var_obj.get('key'), "value": var_obj.get('value'),
+            "masked": var_obj.get('masked'), "protected": var_obj.get('protected'),
+            "variable_type": var_obj.get('variable_type')
         }
-        if environment_scope is not None:
-            var["environment_scope"] = environment_scope
-        return self.project.variables.create(var)
 
-    def update_variable(self, key, var, value, masked, protected, variable_type, environment_scope):
-        if (var.value == value and var.protected == protected and var.masked == masked
-                and var.variable_type == variable_type
-                and (var.environment_scope == environment_scope or environment_scope is None)):
-            return False
+        if var_obj.get('environment_scope') is not None:
+            var["environment_scope"] = var_obj.get('environment_scope')
 
-        if self._module.check_mode:
-            return True
-
-        if (var.protected == protected and var.masked == masked
-                and var.variable_type == variable_type
-                and (var.environment_scope == environment_scope or environment_scope is None)):
-            var.value = value
-            var.save()
-            return True
-
-        self.delete_variable(key)
-        self.create_variable(key, value, masked, protected, variable_type, environment_scope)
+        self.project.variables.create(var)
         return True
 
-    def delete_variable(self, key):
+    def update_variable(self, var_obj):
         if self._module.check_mode:
-            return
-        return self.project.variables.delete(key)
+            return True
+        self.delete_variable(var_obj)
+        self.create_variable(var_obj)
+        return True
+
+    def delete_variable(self, var_obj):
+        if self._module.check_mode:
+            return True
+        self.project.variables.delete(var_obj.get('key'), filter={'environment_scope': var_obj.get('environment_scope')})
+        return True
 
 
-def native_python_main(this_gitlab, purge, var_list, state, module):
+def compare(requested_variables, existing_variables, state):
+    # we need to do this, because it was determined in a previous version - more or less buggy
+    # basically it is not necessary and might results in more/other bugs!
+    # but it is required  and only relevant for check mode!!
+    # logic represents state 'present' when not purge. all other can be derived from that
+    # untouched => equal in both
+    # updated => name and scope are equal
+    # added => name and scope does not exist
+    untouched = list()
+    updated = list()
+    added = list()
+
+    if state == 'present':
+        existing_key_scope_vars = list()
+        for item in existing_variables:
+            existing_key_scope_vars.append({'key': item.get('key'), 'environment_scope': item.get('environment_scope')})
+
+        for var in requested_variables:
+            if var in existing_variables:
+                untouched.append(var)
+            else:
+                compare_item = {'key': var.get('name'), 'environment_scope': var.get('environment_scope')}
+                if compare_item in existing_key_scope_vars:
+                    updated.append(var)
+                else:
+                    added.append(var)
+
+    return untouched, updated, added
+
+
+def native_python_main(this_gitlab, purge, requested_variables, state, module):
 
     change = False
     return_value = dict(added=list(), updated=list(), removed=list(), untouched=list())
 
     gitlab_keys = this_gitlab.list_all_project_variables()
-    existing_variables = [x.get_id() for x in gitlab_keys]
+    before = [x.attributes for x in gitlab_keys]
 
-    for key in var_list:
+    gitlab_keys = this_gitlab.list_all_project_variables()
+    existing_variables = [x.attributes for x in gitlab_keys]
 
-        if isinstance(var_list[key], string_types) or isinstance(var_list[key], (integer_types, float)):
-            value = var_list[key]
-            masked = False
-            protected = False
-            variable_type = 'env_var'
-            environment_scope = None
-        elif isinstance(var_list[key], dict):
-            value = var_list[key].get('value')
-            masked = var_list[key].get('masked', False)
-            protected = var_list[key].get('protected', False)
-            variable_type = var_list[key].get('variable_type', 'env_var')
-            environment_scope = var_list[key].get('environment_scope')
-        else:
-            module.fail_json(msg="value must be of type string, integer or dict")
+    # preprocessing:filter out and enrich before compare
+    for item in existing_variables:
+        item.pop('project_id')
 
-        if key in existing_variables:
-            index = existing_variables.index(key)
-            existing_variables[index] = None
+    for item in requested_variables:
+        item['key'] = item.pop('name')
+        item['value'] = str(item.get('value'))
+        if item.get('protected') is None:
+            item['protected'] = False
+        if item.get('masked') is None:
+            item['masked'] = False
+        if item.get('environment_scope') is None:
+            item['environment_scope'] = '*'
+        if item.get('variable_type') is None:
+            item['variable_type'] = 'env_var'
 
-            if state == 'present':
-                single_change = this_gitlab.update_variable(key,
-                                                            gitlab_keys[index],
-                                                            value, masked,
-                                                            protected,
-                                                            variable_type,
-                                                            environment_scope)
-                change = single_change or change
-                if single_change:
-                    return_value['updated'].append(key)
-                else:
-                    return_value['untouched'].append(key)
+    if module.check_mode:
+        untouched, updated, added = compare(requested_variables, existing_variables, state)
 
-            elif state == 'absent':
-                this_gitlab.delete_variable(key)
-                change = True
-                return_value['removed'].append(key)
+    if state == 'present':
+        add_or_update = [x for x in requested_variables if x not in existing_variables]
+        for item in add_or_update:
+            try:
+                if this_gitlab.create_variable(item):
+                    return_value['added'].append(item)
 
-        elif key not in existing_variables and state == 'present':
-            this_gitlab.create_variable(key, value, masked, protected, variable_type, environment_scope)
-            change = True
-            return_value['added'].append(key)
+            except Exception:
+                if this_gitlab.update_variable(item):
+                    return_value['updated'].append(item)
 
-    existing_variables = list(filter(None, existing_variables))
-    if purge:
+        if purge:
+            # refetch and filter
+            gitlab_keys = this_gitlab.list_all_project_variables()
+            existing_variables = [x.attributes for x in gitlab_keys]
+            for item in existing_variables:
+                item.pop('project_id')
+
+            remove = [x for x in existing_variables if x not in requested_variables]
+            for item in remove:
+                if this_gitlab.delete_variable(item):
+                    return_value['removed'].append(item)
+
+    elif state == 'absent':
+        # value does not matter on removing variables.
+        # key and environment scope are sufficient
         for item in existing_variables:
-            this_gitlab.delete_variable(item)
-            change = True
-            return_value['removed'].append(item)
-    else:
-        return_value['untouched'].extend(existing_variables)
+            item.pop('value')
+            item.pop('variable_type')
+        for item in requested_variables:
+            item.pop('value')
+            item.pop('variable_type')
 
-    return change, return_value
+        if not purge:
+            remove_requested = [x for x in requested_variables if x in existing_variables]
+            for item in remove_requested:
+                if this_gitlab.delete_variable(item):
+                    return_value['removed'].append(item)
+
+        else:
+            for item in existing_variables:
+                if this_gitlab.delete_variable(item):
+                    return_value['removed'].append(item)
+
+    if module.check_mode:
+        return_value = dict(added=added, updated=updated, removed=return_value['removed'], untouched=untouched)
+
+    if return_value['added'] or return_value['removed'] or return_value['updated']:
+        change = True
+
+    gitlab_keys = this_gitlab.list_all_project_variables()
+    after = [x.attributes for x in gitlab_keys]
+
+    return change, return_value, before, after
 
 
 def main():
@@ -271,7 +401,15 @@ def main():
         project=dict(type='str', required=True),
         purge=dict(type='bool', required=False, default=False),
         vars=dict(type='dict', required=False, default=dict(), no_log=True),
-        state=dict(type='str', default="present", choices=["absent", "present"])
+        variables=dict(type='list', elements='dict', required=False, default=list(), options=dict(
+            name=dict(type='str', required=True),
+            value=dict(type='str', required=True, no_log=True),
+            masked=dict(type='bool', default=False),
+            protected=dict(type='bool', default=False),
+            environment_scope=dict(type='str', default='*'),
+            variable_type=dict(type='str', default='env_var', choices=["env_var", "file"])
+        )),
+        state=dict(type='str', default="present", choices=["absent", "present"]),
     )
 
     module = AnsibleModule(
@@ -282,6 +420,7 @@ def main():
             ['api_username', 'api_job_token'],
             ['api_token', 'api_oauth_token'],
             ['api_token', 'api_job_token'],
+            ['vars', 'variables'],
         ],
         required_together=[
             ['api_username', 'api_password'],
@@ -294,6 +433,12 @@ def main():
 
     purge = module.params['purge']
     var_list = module.params['vars']
+
+    if var_list:
+        variables = vars_to_variables(var_list, module)
+    else:
+        variables = module.params['variables']
+
     state = module.params['state']
 
     if not HAS_GITLAB_PACKAGE:
@@ -303,7 +448,26 @@ def main():
 
     this_gitlab = GitlabProjectVariables(module=module, gitlab_instance=gitlab_instance)
 
-    change, return_value = native_python_main(this_gitlab, purge, var_list, state, module)
+    change, raw_return_value, before, after = native_python_main(this_gitlab, purge, variables, state, module)
+
+    # postprocessing
+    for item in after:
+        item.pop('project_id')
+        item['name'] = item.pop('key')
+    for item in before:
+        item.pop('project_id')
+        item['name'] = item.pop('key')
+
+    untouched_key_name = 'key'
+    if not module.check_mode:
+        untouched_key_name = 'name'
+        raw_return_value['untouched'] = [x for x in before if x in after]
+
+    added = [x.get('key') for x in raw_return_value['added']]
+    updated = [x.get('key') for x in raw_return_value['updated']]
+    removed = [x.get('key') for x in raw_return_value['removed']]
+    untouched = [x.get(untouched_key_name) for x in raw_return_value['untouched']]
+    return_value = dict(added=added, updated=updated, removed=removed, untouched=untouched)
 
     module.exit_json(changed=change, project_variable=return_value)
 

tests/integration/targets/gitlab_project_variable/tasks/main.yml

@@ -35,8 +35,9 @@
     api_url: "{{ gitlab_host }}"
     api_token: "{{ gitlab_login_token }}"
     project: "{{ gitlab_project_name }}"
-    vars:
-      ACCESS_KEY_ID: checkmode
+    variables:
+      - name: ACCESS_KEY_ID
+        value: checkmode
   register: gitlab_project_variable_state
 
 - name: state must be changed
@@ -455,8 +456,8 @@
     api_url: "{{ gitlab_host }}"
     api_token: "{{ gitlab_login_token }}"
     project: "{{ gitlab_project_name }}"
-    vars:
-      my_test_var:
+    variables:
+      - name: my_test_var
         value: my_test_value
         variable_type: file
     purge: False
@@ -470,7 +471,8 @@
       - gitlab_project_variable_state.project_variable.untouched|length == 0
       - gitlab_project_variable_state.project_variable.removed|length == 0
       - gitlab_project_variable_state.project_variable.updated|length == 0
-      - gitlab_project_variable_state.project_variable.added[0] == "my_test_var"
+      # VALUE_SPECIFIED_IN_NO_LOG_PARAMETER
+      #- gitlab_project_variable_state.project_variable.added[0] == "my_test_var"
 
 - name: change variable_type attribute
   gitlab_project_variable:
@@ -612,3 +614,33 @@
       - gitlab_project_variable_state.project_variable.untouched|length == 0
       - gitlab_project_variable_state.project_variable.removed|length == 40
       - gitlab_project_variable_state.project_variable.updated|length == 0
+
+- name: same vars, diff scope
+  gitlab_project_variable:
+    api_url: "{{ gitlab_host }}"
+    api_token: "{{ gitlab_login_token }}"
+    project: "{{ gitlab_project_name }}"
+    purge: True
+    variables:
+      - name: SECRET_ACCESS_KEY
+        value: 3214cbad
+        masked: true
+        protected: true
+        variable_type: env_var
+        environment_scope: production
+      - name: SECRET_ACCESS_KEY
+        value: hello_world
+        masked: true
+        protected: true
+        variable_type: env_var
+        environment_scope: development
+  register: gitlab_project_variable_state
+
+- name: verify two vars
+  assert:
+    that:
+      - gitlab_project_variable_state.changed
+      - gitlab_project_variable_state.project_variable.added|length == 2
+      - gitlab_project_variable_state.project_variable.untouched|length == 0
+      - gitlab_project_variable_state.project_variable.removed|length == 0
+      - gitlab_project_variable_state.project_variable.updated|length == 0