respect new variable property in gitlab_group_variable and gitlab_project_variable (#5667)

* draft

* add changelog fragment

* rework

* rework group variables

* add new line at end of file

* Update plugins/module_utils/gitlab.py

Co-authored-by: Nejc Habjan <hab.nejc@gmail.com>

* rename

* revert

* return a copy

* Update plugins/modules/gitlab_project_variable.py

Co-authored-by: Felix Fontein <felix@fontein.de>

Co-authored-by: Nejc Habjan <hab.nejc@gmail.com>
Co-authored-by: Felix Fontein <felix@fontein.de>

changelogs/fragments/5666-gitlab-variables.yml

@@ -0,0 +1,3 @@
+bugfixes:
+  - gitlab_group_variables - fix dropping variables accidentally when GitLab introduced new properties (https://github.com/ansible-collections/community.general/pull/5667).
+  - gitlab_project_variables - fix dropping variables accidentally when GitLab introduced new properties (https://github.com/ansible-collections/community.general/pull/5667).

plugins/module_utils/gitlab.py

@@ -110,3 +110,14 @@ def gitlab_authentication(module):
             GitLab remove Session API now that private tokens are removed from user API endpoints since version 10.2." % to_native(e))
 
     return gitlab_instance
+
+
+def filter_returned_variables(gitlab_variables):
+    # pop properties we don't know
+    existing_variables = [dict(x.attributes) for x in gitlab_variables]
+    KNOWN = ['key', 'value', 'masked', 'protected', 'variable_type', 'environment_scope']
+    for item in existing_variables:
+        for key in list(item.keys()):
+            if key not in KNOWN:
+                item.pop(key)
+    return existing_variables

plugins/modules/gitlab_group_variable.py

@@ -165,7 +165,7 @@ from ansible.module_utils.six import string_types
 from ansible.module_utils.six import integer_types
 
 from ansible_collections.community.general.plugins.module_utils.gitlab import (
-    auth_argument_spec, gitlab_authentication, ensure_gitlab_package
+    auth_argument_spec, gitlab_authentication, ensure_gitlab_package, filter_returned_variables
 )
 
 
@@ -296,11 +296,7 @@ def native_python_main(this_gitlab, purge, requested_variables, state, module):
     before = [x.attributes for x in gitlab_keys]
 
     gitlab_keys = this_gitlab.list_all_group_variables()
-    existing_variables = [x.attributes for x in gitlab_keys]
+    existing_variables = filter_returned_variables(gitlab_keys)
-
-    # preprocessing:filter out and enrich before compare
-    for item in existing_variables:
-        item.pop('group_id')
 
     for item in requested_variables:
         item['key'] = item.pop('name')
@@ -331,9 +327,7 @@ def native_python_main(this_gitlab, purge, requested_variables, state, module):
         if purge:
             # refetch and filter
             gitlab_keys = this_gitlab.list_all_group_variables()
-            existing_variables = [x.attributes for x in gitlab_keys]
+            existing_variables = filter_returned_variables(gitlab_keys)
-            for item in existing_variables:
-                item.pop('group_id')
 
             remove = [x for x in existing_variables if x not in requested_variables]
             for item in remove:

plugins/modules/gitlab_project_variable.py

@@ -189,7 +189,7 @@ except Exception:
     HAS_GITLAB_PACKAGE = False
 
 from ansible_collections.community.general.plugins.module_utils.gitlab import (
-    auth_argument_spec, gitlab_authentication, ensure_gitlab_package
+    auth_argument_spec, gitlab_authentication, ensure_gitlab_package, filter_returned_variables
 )
 
 
@@ -255,9 +255,11 @@ class GitlabProjectVariables(object):
             return True
 
         var = {
-            "key": var_obj.get('key'), "value": var_obj.get('value'),
+            "key": var_obj.get('key'),
-            "masked": var_obj.get('masked'), "protected": var_obj.get('protected'),
+            "value": var_obj.get('value'),
-            "variable_type": var_obj.get('variable_type')
+            "masked": var_obj.get('masked'),
+            "protected": var_obj.get('protected'),
+            "variable_type": var_obj.get('variable_type'),
         }
 
         if var_obj.get('environment_scope') is not None:
@@ -319,12 +321,9 @@ def native_python_main(this_gitlab, purge, requested_variables, state, module):
     before = [x.attributes for x in gitlab_keys]
 
     gitlab_keys = this_gitlab.list_all_project_variables()
-    existing_variables = [x.attributes for x in gitlab_keys]
+    existing_variables = filter_returned_variables(gitlab_keys)
-
-    # preprocessing:filter out and enrich before compare
-    for item in existing_variables:
-        item.pop('project_id')
 
+    # filter out and enrich before compare
     for item in requested_variables:
         item['key'] = item.pop('name')
         item['value'] = str(item.get('value'))
@@ -354,9 +353,7 @@ def native_python_main(this_gitlab, purge, requested_variables, state, module):
         if purge:
             # refetch and filter
             gitlab_keys = this_gitlab.list_all_project_variables()
-            existing_variables = [x.attributes for x in gitlab_keys]
+            existing_variables = filter_returned_variables(gitlab_keys)
-            for item in existing_variables:
-                item.pop('project_id')
 
             remove = [x for x in existing_variables if x not in requested_variables]
             for item in remove:
@@ -409,7 +406,7 @@ def main():
             masked=dict(type='bool', default=False),
             protected=dict(type='bool', default=False),
             environment_scope=dict(type='str', default='*'),
-            variable_type=dict(type='str', default='env_var', choices=["env_var", "file"])
+            variable_type=dict(type='str', default='env_var', choices=["env_var", "file"]),
         )),
         state=dict(type='str', default="present", choices=["absent", "present"]),
     )