Fortios file only mode + integration tests (#23275)

* WIP file_mode

* WIP

* Add file_mode + integration tests

* fix pep8

* Update doc fragments
Create mutualy_exclusive param
Fix yamllint problem in tests

* Add aliases file + main playbook for fortios

* Install pyfg before running tests

* Install pyfg before running tests in role

* Remove pre_task as it's done in roles

* Force pyFG minimal version for python3

* role_path not role_dir :(

* Change requirements

* Specify Error type when error on import

* Bug in pygf library with python 2.5 (PR is waiting https://github.com/spotify/pyfg/pull/19)

* Bad requirement format

* still bad format -_-'

* remove test/integration/fortios.py (auto generated by tests)
missing new lines at end of file

* pyFG is now fixed in 0.50

lib/ansible/module_utils/fortios.py

@@ -38,13 +38,15 @@ try:
     from pyFG import FortiOS, FortiConfig
     from pyFG.exceptions import CommandExecutionException, FailedCommit
     HAS_PYFG=True
-except:
+except ImportError:
     HAS_PYFG=False
 
 fortios_argument_spec = dict(
-    host            = dict(required=True ),
+    file_mode       = dict(type='bool', default=False),
-    username        = dict(required=True ),
+    config_file     = dict(type='path'),
-    password        = dict(required=True, type='str', no_log=True ),
+    host            = dict( ),
+    username        = dict( ),
+    password        = dict(type='str', no_log=True ),
     timeout         = dict(type='int', default=60),
     vdom            = dict(type='str', default=None ),
     backup          = dict(type='bool', default=False),
@@ -53,9 +55,16 @@ fortios_argument_spec = dict(
 )
 
 fortios_required_if = [
+    ['file_mode', False, ['host', 'username', 'password']],
+    ['file_mode', True, ['config_file']],
     ['backup',   True   , ['backup_path']   ],
 ]
 
+fortios_mutually_exclusive = [
+    ['config_file', 'host'],
+    ['config_file', 'username'],
+    ['config_file', 'password']
+]
 
 fortios_error_codes = {
     '-3':"Object not found",
@@ -96,38 +105,54 @@ class AnsibleFortios(object):
 
 
     def _connect(self):
-        host = self.module.params['host']
+        if self.module.params['file_mode']:
-        username = self.module.params['username']
+            self.forti_device = FortiOS('')
-        password = self.module.params['password']
+        else:
-        timeout = self.module.params['timeout']
+            host = self.module.params['host']
-        vdom = self.module.params['vdom']
+            username = self.module.params['username']
+            password = self.module.params['password']
+            timeout = self.module.params['timeout']
+            vdom = self.module.params['vdom']
 
-        self.forti_device = FortiOS(host, username=username, password=password, timeout=timeout, vdom=vdom)
+            self.forti_device = FortiOS(host, username=username, password=password, timeout=timeout, vdom=vdom)
 
-        try:
+            try:
-            self.forti_device.open()
+                self.forti_device.open()
-        except Exception:
+            except Exception:
-            e = get_exception()
+                e = get_exception()
-            self.module.fail_json(msg='Error connecting device. %s' % e)
+                self.module.fail_json(msg='Error connecting device. %s' % e)
 
 
     def load_config(self, path):
-        self._connect()
         self.path = path
-        #get  config
+        self._connect()
-        try:
+        #load in file_mode
-            self.forti_device.load_config(path=path)
+        if self.module.params['file_mode']:
-            self.result['running_config'] = self.forti_device.running_config.to_text()
+            try:
-        except Exception:
+                f = open(self.module.params['config_file'], 'r')
-            self.forti_device.close()
+                running = f.read()
-            e = get_exception()
+                f.close()
-            self.module.fail_json(msg='Error reading running config. %s' % e)
+            except IOError:
+                e = get_exception()
+                self.module.fail_json(msg='Error reading configuration file. %s' % e)
+            self.forti_device.load_config(config_text=running, path = path)
+
+        else:
+            #get  config
+            try:
+                self.forti_device.load_config(path=path)
+            except Exception:
+                self.forti_device.close()
+                e = get_exception()
+                self.module.fail_json(msg='Error reading running config. %s' % e)
+
+        #set configs in object
+        self.result['running_config'] = self.forti_device.running_config.to_text()
+        self.candidate_config = self.forti_device.candidate_config
 
         #backup if needed
         if self.module.params['backup']:
-            backup(self.module, self.result['running_config'])
+            backup(self.module, self.forti_device.running_config.to_text())
-
-        self.candidate_config = self.forti_device.candidate_config
 
 
     def apply_changes(self):
@@ -138,16 +163,25 @@ class AnsibleFortios(object):
 
         #Commit if not check mode
         if change_string and not self.module.check_mode:
-            try:
+            if self.module.params['file_mode']:
-                self.forti_device.commit()
+                try:
-            except FailedCommit:
+                    f = open(self.module.params['config_file'], 'w')
-                #Something's wrong (rollback is automatic)
+                    f.write(self.candidate_config.to_text())
-                self.forti_device.close()
+                    f.close
-                e = get_exception()
+                except IOError:
-                error_list = self.get_error_infos(e)
+                    e = get_exception()
-                self.module.fail_json(msg_error_list=error_list, msg="Unable to commit change, check your args, the error was %s" % e.message )
+                    self.module.fail_json(msg='Error writing configuration file. %s' % e)
+            else:
+                try:
+                    self.forti_device.commit()
+                except FailedCommit:
+                    #Something's wrong (rollback is automatic)
+                    self.forti_device.close()
+                    e = get_exception()
+                    error_list = self.get_error_infos(e)
+                    self.module.fail_json(msg_error_list=error_list, msg="Unable to commit change, check your args, the error was %s" % e.message )
 
-        self.forti_device.close()
+                self.forti_device.close()
         self.module.exit_json(**self.result)
 
 

lib/ansible/utils/module_docs_fragments/fortios.py

@@ -23,18 +23,25 @@ class ModuleDocFragment(object):
     # Standard files documentation fragment
     DOCUMENTATION = """
 options:
+  file_mode:
+    description:
+      - Don't connect to any device, only use I(config_file) as input and Output.
+    default: false
+    type: bool
+    version_added: "2.4"
+  config_file:
+    description:
+      - Path to configuration file. Required when I(file_mode) is True.
+    version_added: "2.4"
   host:
     description:
-      - Specifies the DNS hostname or IP address for connecting to the remote fortios device.
+      - Specifies the DNS hostname or IP address for connecting to the remote fortios device. Required when I(file_mode) is False.
-    required: true
   username:
     description:
-      - Configures the username used to authenticate to the remote device.
+      - Configures the username used to authenticate to the remote device. Required when I(file_mode) is True.
-    required: true
   password:
     description:
-      - Specifies the password used to authenticate to the remote device.
+      - Specifies the password used to authenticate to the remote device. Required when I(file_mode) is True.
-    required: true
   timeout:
     description:
       - Timeout in seconds for connecting to the remote device.

test/integration/targets/fortios_ipv4_policy/aliases

@@ -0,0 +1 @@
+posix/ci/group1

test/integration/targets/fortios_ipv4_policy/files/requirements.txt

@@ -0,0 +1 @@
+pyfg>=0.50

test/integration/targets/fortios_ipv4_policy/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: install required libraries
+  pip:
+    requirements: "{{ role_path }}/files/requirements.txt"
+
+- { include: test_indempotency.yml }
+- { include: test_params.yml }

test/integration/targets/fortios_ipv4_policy/tasks/test_indempotency.yml

@@ -0,0 +1,68 @@
+---
+ - name: Add policy
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: present
+   register: add_policy
+
+ - name: Assert
+   assert:
+     that:
+      - "add_policy.changed == true"
+
+ - name: Add existing policy 
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: present
+   register: add_policy
+
+ - name: Assert
+   assert:
+     that:
+      - "add_policy.changed == false"
+
+ - name: Delete existing policy
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: absent
+   register: del_policy
+
+ - name: Assert
+   assert:
+     that:
+      - "del_policy.changed == true"
+
+ - name: Delete not-existing policy
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: absent
+   register: del_policy
+
+ - name: Assert
+   assert:
+     that:
+      - "del_policy.changed == false"

test/integration/targets/fortios_ipv4_policy/tasks/test_params.yml

@@ -0,0 +1,74 @@
+---
+ - name: Forget id
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     # id: 42
+     src_addr: all
+     dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: present
+   register: forget_id
+   ignore_errors: True
+
+ - name: Forget src_addr
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     # src_addr: all
+     dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: present
+   register: forget_src_addr
+   ignore_errors: True
+
+ - name: Forget dst_addr
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     # dst_addr: all
+     policy_action: accept
+     service: ALL
+     state: present
+   register: forget_dst_addr
+   ignore_errors: True
+
+ - name: Forget policy_action
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     dst_addr: all
+     # policy_action: accept
+     service: ALL
+     state: present
+   register: forget_policy_action
+   ignore_errors: True
+
+ - name: Forget service
+   fortios_ipv4_policy:
+     file_mode: true
+     config_file: "{{role_path}}/files/default_config.conf"
+     id: 42
+     src_addr: all
+     dst_addr: all
+     policy_action: accept
+     # service: ALL
+     state: present
+   register: forget_service
+   ignore_errors: True
+
+ - name: Verify that all previous test have failed
+   assert:
+     that:
+      - "forget_id.failed == True"
+      - "forget_src_addr.failed == True"
+      - "forget_dst_addr.failed == True"
+      - "forget_policy_action.failed == True"
+      - "forget_service.failed == True"