From e4a25beedcbc6cafb3b67cc26729a7cb2592c960 Mon Sep 17 00:00:00 2001 From: Kamil Markowicz Date: Wed, 13 Apr 2022 07:16:54 -0400 Subject: [PATCH] Terraform init -upgrade flag (#4455) * Adds optional `-upgrade` flag to terraform init. This allows Terraform to install provider dependencies into an existing project when the provider constraints change. * fix transposed documentation keys * Add integration tests for terraform init * Revert to validate_certs: yes for general public testing * skip integration tests on irrelevant platforms * skip legacy Python versions from CI tests * add changelog fragment * Update plugins/modules/cloud/misc/terraform.py Adds version_added metadata to the new module option. Co-authored-by: Felix Fontein * Change terraform_arch constant to Ansible fact mapping * correct var typo, clarify task purpose * Squashed some logic bugs, added override for local Terraform If `existing_terraform_path` is provided, the playbook will not download Terraform or check its version. I also tested this on a local system with Terraform installed, and squashed some bugs related to using of an existing binary. * revert to previous test behavior for TF install * readability cleanup * Update plugins/modules/cloud/misc/terraform.py Co-authored-by: Felix Fontein --- .../4455-terraform-provider-upgrade.yml | 3 + plugins/modules/cloud/misc/terraform.py | 14 +++- .../integration/targets/terraform/.gitignore | 4 ++ tests/integration/targets/terraform/aliases | 7 ++ .../targets/terraform/meta/main.yml | 3 + .../targets/terraform/tasks/main.yml | 70 +++++++++++++++++++ .../terraform/tasks/test_provider_upgrade.yml | 23 ++++++ .../templates/provider_test/main.tf.j2 | 8 +++ .../targets/terraform/vars/main.yml | 37 ++++++++++ 9 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/4455-terraform-provider-upgrade.yml create mode 100644 tests/integration/targets/terraform/.gitignore create mode 100644 tests/integration/targets/terraform/aliases create mode 100644 tests/integration/targets/terraform/meta/main.yml create mode 100644 tests/integration/targets/terraform/tasks/main.yml create mode 100644 tests/integration/targets/terraform/tasks/test_provider_upgrade.yml create mode 100644 tests/integration/targets/terraform/templates/provider_test/main.tf.j2 create mode 100644 tests/integration/targets/terraform/vars/main.yml diff --git a/changelogs/fragments/4455-terraform-provider-upgrade.yml b/changelogs/fragments/4455-terraform-provider-upgrade.yml new file mode 100644 index 0000000000..5fe2d08ca7 --- /dev/null +++ b/changelogs/fragments/4455-terraform-provider-upgrade.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - terraform - adds ``terraform_upgrade`` parameter which allows ``terraform init`` to satisfy new provider constraints in an existing Terraform project (https://github.com/ansible-collections/community.general/issues/4333). \ No newline at end of file diff --git a/plugins/modules/cloud/misc/terraform.py b/plugins/modules/cloud/misc/terraform.py index 8eca14e712..3c3662c6d2 100644 --- a/plugins/modules/cloud/misc/terraform.py +++ b/plugins/modules/cloud/misc/terraform.py @@ -124,6 +124,12 @@ options: type: list elements: path version_added: '0.2.0' + provider_upgrade: + description: + - Allows Terraform init to upgrade providers to versions specified in the project's version constraints. + default: false + type: bool + version_added: 4.8.0 init_reconfigure: description: - Forces backend reconfiguration during init. @@ -266,7 +272,7 @@ def _state_args(state_file): return [] -def init_plugins(bin_path, project_path, backend_config, backend_config_files, init_reconfigure, plugin_paths): +def init_plugins(bin_path, project_path, backend_config, backend_config_files, init_reconfigure, provider_upgrade, plugin_paths): command = [bin_path, 'init', '-input=false'] if backend_config: for key, val in backend_config.items(): @@ -279,6 +285,8 @@ def init_plugins(bin_path, project_path, backend_config, backend_config_files, i command.extend(['-backend-config', f]) if init_reconfigure: command.extend(['-reconfigure']) + if provider_upgrade: + command.extend(['-upgrade']) if plugin_paths: for plugin_path in plugin_paths: command.extend(['-plugin-dir', plugin_path]) @@ -384,6 +392,7 @@ def main(): overwrite_init=dict(type='bool', default=True), check_destroy=dict(type='bool', default=False), parallelism=dict(type='int'), + provider_upgrade=dict(type='bool', default=False), ), required_if=[('state', 'planned', ['plan_file'])], supports_check_mode=True, @@ -405,6 +414,7 @@ def main(): init_reconfigure = module.params.get('init_reconfigure') overwrite_init = module.params.get('overwrite_init') check_destroy = module.params.get('check_destroy') + provider_upgrade = module.params.get('provider_upgrade') if bin_path is not None: command = [bin_path] @@ -422,7 +432,7 @@ def main(): if force_init: if overwrite_init or not os.path.isfile(os.path.join(project_path, ".terraform", "terraform.tfstate")): - init_plugins(command[0], project_path, backend_config, backend_config_files, init_reconfigure, plugin_paths) + init_plugins(command[0], project_path, backend_config, backend_config_files, init_reconfigure, provider_upgrade, plugin_paths) workspace_ctx = get_workspace_context(command[0], project_path) if workspace_ctx["current"] != workspace: diff --git a/tests/integration/targets/terraform/.gitignore b/tests/integration/targets/terraform/.gitignore new file mode 100644 index 0000000000..45b67fdab4 --- /dev/null +++ b/tests/integration/targets/terraform/.gitignore @@ -0,0 +1,4 @@ +**/.terraform/* +*.tfstate +*.tfstate.* +.terraform.lock.hcl \ No newline at end of file diff --git a/tests/integration/targets/terraform/aliases b/tests/integration/targets/terraform/aliases new file mode 100644 index 0000000000..71a9c13cf1 --- /dev/null +++ b/tests/integration/targets/terraform/aliases @@ -0,0 +1,7 @@ +shippable/posix/group1 +skip/windows +skip/aix +skip/osx +skip/macos +skip/freebsd +skip/python2 \ No newline at end of file diff --git a/tests/integration/targets/terraform/meta/main.yml b/tests/integration/targets/terraform/meta/main.yml new file mode 100644 index 0000000000..56bc554611 --- /dev/null +++ b/tests/integration/targets/terraform/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - setup_pkg_mgr + - setup_remote_tmp_dir diff --git a/tests/integration/targets/terraform/tasks/main.yml b/tests/integration/targets/terraform/tasks/main.yml new file mode 100644 index 0000000000..f8ac3a919d --- /dev/null +++ b/tests/integration/targets/terraform/tasks/main.yml @@ -0,0 +1,70 @@ +--- + + +# This block checks and registers Terraform version of the binary found in path. + +- name: Check for existing Terraform in path + block: + - name: Check if terraform is present in path + command: "command -v terraform" + register: terraform_binary_path + ignore_errors: true + + - name: Check Terraform version + command: terraform version + register: terraform_version_output + when: terraform_binary_path.rc == 0 + + - name: Set terraform version + set_fact: + terraform_version_installed: "{{ terraform_version_output.stdout | regex_search('(?!Terraform.*v)([0-9]+\\.[0-9]+\\.[0-9]+)') }}" + when: terraform_version_output.changed + +# This block handles the tasks of installing the Terraform binary. This happens if there is no existing +# terraform in $PATH OR version does not match `terraform_version`. + +- name: Execute Terraform install tasks + block: + + - name: Install Terraform + debug: + msg: "Installing terraform {{ terraform_version }}, found: {{ terraform_version_installed | default('no terraform binary found') }}." + + - name: Ensure unzip is present + ansible.builtin.package: + name: unzip + state: present + + - name: Install Terraform binary + unarchive: + src: "{{ terraform_url }}" + dest: "{{ remote_tmp_dir }}" + mode: 0755 + remote_src: yes + validate_certs: "{{ validate_certs }}" + + when: terraform_version_installed is not defined or terraform_version_installed != terraform_version + +# This sets `terraform_binary_path` to coalesced output of first non-empty string in this order: +# path from the 'Check if terraform is present in path' task, and lastly, the fallback path. + +- name: Set path to terraform binary + set_fact: + terraform_binary_path: "{{ terraform_binary_path.stdout or remote_tmp_dir ~ '/terraform' }}" + +- name: Create terraform project directory + file: + path: "{{ terraform_project_dir }}/{{ item['name'] }}" + state: directory + mode: 0755 + loop: "{{ terraform_provider_versions }}" + loop_control: + index_var: provider_index + +- name: Loop over provider upgrade test tasks + include_tasks: test_provider_upgrade.yml + vars: + tf_provider: "{{ terraform_provider_versions[provider_index] }}" + loop: "{{ terraform_provider_versions }}" + loop_control: + index_var: provider_index \ No newline at end of file diff --git a/tests/integration/targets/terraform/tasks/test_provider_upgrade.yml b/tests/integration/targets/terraform/tasks/test_provider_upgrade.yml new file mode 100644 index 0000000000..26cc0d842a --- /dev/null +++ b/tests/integration/targets/terraform/tasks/test_provider_upgrade.yml @@ -0,0 +1,23 @@ +--- + +- name: Output terraform provider test project + ansible.builtin.template: + src: templates/provider_test/main.tf.j2 + dest: "{{ terraform_project_dir }}/{{ tf_provider['name'] }}/main.tf" + force: yes + register: terraform_provider_hcl + +# The purpose of this task is to init terraform multiple times with different provider module +# versions, so that we can verify that provider upgrades during init work as intended. + +- name: Init Terraform configuration with pinned provider version + community.general.terraform: + project_path: "{{ terraform_provider_hcl.dest | dirname }}" + binary_path: "{{ terraform_binary_path }}" + force_init: yes + provider_upgrade: "{{ terraform_provider_upgrade }}" + state: present + register: terraform_init_result + +- assert: + that: terraform_init_result is not failed \ No newline at end of file diff --git a/tests/integration/targets/terraform/templates/provider_test/main.tf.j2 b/tests/integration/targets/terraform/templates/provider_test/main.tf.j2 new file mode 100644 index 0000000000..58eb3a24bb --- /dev/null +++ b/tests/integration/targets/terraform/templates/provider_test/main.tf.j2 @@ -0,0 +1,8 @@ +terraform { + required_providers { + {{ tf_provider['name'] }} = { + source = "{{ tf_provider['source'] }}" + version = "{{ tf_provider['version'] }}" + } + } +} \ No newline at end of file diff --git a/tests/integration/targets/terraform/vars/main.yml b/tests/integration/targets/terraform/vars/main.yml new file mode 100644 index 0000000000..64307e889d --- /dev/null +++ b/tests/integration/targets/terraform/vars/main.yml @@ -0,0 +1,37 @@ +--- + +# Terraform version that will be downloaded +terraform_version: 1.1.7 + +# Architecture of the downloaded Terraform release (needs to match target testing platform) + +terraform_arch: "{{ ansible_system | lower }}_{{terraform_arch_map[ansible_architecture] }}" + +# URL of where the Terraform binary will be downloaded from +terraform_url: "https://releases.hashicorp.com/terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_{{ terraform_arch }}.zip" + +# Controls whether the unarchive task will validate TLS certs of the Terraform binary host +validate_certs: yes + +# Directory where Terraform tests will be created +terraform_project_dir: "{{ remote_tmp_dir }}/tf_provider_test" + +# Controls whether terraform init will use the `-upgrade` flag +terraform_provider_upgrade: yes + +# list of dicts containing Terraform providers that will be tested +# The null provider is a good candidate, as it's small and has no external dependencies +terraform_provider_versions: + - name: "null" + source: "hashicorp/null" + version: ">=2.0.0, < 3.0.0" + - name: "null" + source: "hashicorp/null" + version: ">=3.0.0" + +# mapping between values returned from ansible_architecture and arch names used by golang builds of Terraform +# see https://www.terraform.io/downloads + +terraform_arch_map: + x86_64: amd64 + arm64: arm64