diff --git a/changelogs/fragments/54294-openssl-backup.yaml b/changelogs/fragments/54294-openssl-backup.yaml
new file mode 100644
index 0000000000..9582dbe355
--- /dev/null
+++ b/changelogs/fragments/54294-openssl-backup.yaml
@@ -0,0 +1,6 @@
+minor_changes:
+- "openssl_certificate - add ``backup`` option."
+- "openssl_csr - add ``backup`` option."
+- "openssl_dhparam - add ``backup`` option."
+- "openssl_pkcs12 - add ``backup`` option."
+- "openssl_publickey - add ``backup`` option."
diff --git a/lib/ansible/modules/crypto/openssl_certificate.py b/lib/ansible/modules/crypto/openssl_certificate.py
index baf56d4cc5..c65eef8f03 100644
--- a/lib/ansible/modules/crypto/openssl_certificate.py
+++ b/lib/ansible/modules/crypto/openssl_certificate.py
@@ -28,6 +28,9 @@ description:
     - Many properties that can be specified in this module are for validation of an
       existing or newly generated certificate. The proper place to specify them, if you
       want to receive a certificate with these properties is a CSR (Certificate Signing Request).
+    - "Please note that the module regenerates existing certificate if it doesn't match the module's
+      options, or if it seems to be corrupt. If you are concerned that this could overwrite
+      your existing certificate, consider using the I(backup) option."
     - It uses the pyOpenSSL or cryptography python library to interact with OpenSSL.
     - If both the cryptography and PyOpenSSL libraries are available (and meet the minimum version requirements)
       cryptography will be preferred as a backend over PyOpenSSL (unless the backend is forced with C(select_crypto_backend))
@@ -359,6 +362,15 @@ options:
         choices: [ auto, cryptography, pyopenssl ]
         version_added: "2.8"
 
+    backup:
+        description:
+            - Create a backup file including a timestamp so you can get the original
+              certificate back if you overwrote it with a new one by accident.
+            - This is not used by the C(assertonly) provider.
+        type: bool
+        default: no
+        version_added: "2.8"
+
 extends_documentation_fragment: files
 notes:
     - All ASN.1 TIME values should be specified following the YYYYMMDDHHMMSSZ pattern.
@@ -510,6 +522,11 @@ filename:
     returned: changed or success
     type: str
     sample: /etc/ssl/crt/www.ansible.com.crt
+backup_file:
+    description: Name of backup file created.
+    returned: changed and if I(backup) is C(yes)
+    type: str
+    sample: /path/to/www.ansible.com.crt.2019-03-09@11:22~
 '''
 
 
@@ -576,6 +593,9 @@ class Certificate(crypto_utils.OpenSSLObject):
         self.backend = backend
         self.module = module
 
+        self.backup = module.params['backup']
+        self.backup_file = None
+
     def get_relative_time_option(self, input_string, input_name):
         """Return an ASN1 formatted string if a relative timespec
            or an ASN1 formatted string is provided."""
@@ -663,6 +683,11 @@ class Certificate(crypto_utils.OpenSSLObject):
                     return False
             return True
 
+    def remove(self, module):
+        if self.backup:
+            self.backup_file = module.backup_local(self.path)
+        super(Certificate, self).remove(module)
+
     def check(self, module, perms_required=True):
         """Ensure the resource is in its desired state."""
 
@@ -711,6 +736,8 @@ class CertificateAbsent(Certificate):
             'privatekey': self.privatekey_path,
             'csr': self.csr_path
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         return result
 
@@ -769,6 +796,8 @@ class SelfSignedCertificateCryptography(Certificate):
 
             self.cert = certificate
 
+            if self.backup:
+                self.backup_file = module.backup_local(self.path)
             crypto_utils.write_file(module, certificate.public_bytes(Encoding.PEM))
             self.changed = True
         else:
@@ -786,6 +815,8 @@ class SelfSignedCertificateCryptography(Certificate):
             'privatekey': self.privatekey_path,
             'csr': self.csr_path
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         if check_mode:
             result.update({
@@ -847,6 +878,8 @@ class SelfSignedCertificate(Certificate):
             cert.sign(self.privatekey, self.digest)
             self.cert = cert
 
+            if self.backup:
+                self.backup_file = module.backup_local(self.path)
             crypto_utils.write_file(module, crypto.dump_certificate(crypto.FILETYPE_PEM, self.cert))
             self.changed = True
 
@@ -862,6 +895,8 @@ class SelfSignedCertificate(Certificate):
             'privatekey': self.privatekey_path,
             'csr': self.csr_path
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         if check_mode:
             result.update({
@@ -935,6 +970,8 @@ class OwnCACertificateCryptography(Certificate):
 
             self.cert = certificate
 
+            if self.backup:
+                self.backup_file = module.backup_local(self.path)
             crypto_utils.write_file(module, certificate.public_bytes(Encoding.PEM))
             self.changed = True
         else:
@@ -954,6 +991,8 @@ class OwnCACertificateCryptography(Certificate):
             'ca_cert': self.ca_cert_path,
             'ca_privatekey': self.ca_privatekey_path
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         if check_mode:
             result.update({
@@ -1024,6 +1063,8 @@ class OwnCACertificate(Certificate):
             cert.sign(self.ca_privatekey, self.digest)
             self.cert = cert
 
+            if self.backup:
+                self.backup_file = module.backup_local(self.path)
             crypto_utils.write_file(module, crypto.dump_certificate(crypto.FILETYPE_PEM, self.cert))
             self.changed = True
 
@@ -1041,6 +1082,8 @@ class OwnCACertificate(Certificate):
             'ca_cert': self.ca_cert_path,
             'ca_privatekey': self.ca_privatekey_path
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         if check_mode:
             result.update({
@@ -1614,6 +1657,8 @@ class AcmeCertificate(Certificate):
                                                             self.csr_path,
                                                             self.challenge_path),
                                          check_rc=True)[1]
+                if self.backup:
+                    self.backup_file = module.backup_local(self.path)
                 crypto_utils.write_file(module, to_bytes(crt))
                 self.changed = True
             except OSError as exc:
@@ -1632,6 +1677,8 @@ class AcmeCertificate(Certificate):
             'accountkey': self.accountkey_path,
             'csr': self.csr_path,
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         return result
 
@@ -1644,6 +1691,7 @@ def main():
             provider=dict(type='str', choices=['acme', 'assertonly', 'ownca', 'selfsigned']),
             force=dict(type='bool', default=False,),
             csr_path=dict(type='path'),
+            backup=dict(type='bool', default=False),
             select_crypto_backend=dict(type='str', default='auto', choices=['auto', 'cryptography', 'pyopenssl']),
 
             # General properties of a certificate
diff --git a/lib/ansible/modules/crypto/openssl_csr.py b/lib/ansible/modules/crypto/openssl_csr.py
index f5e32ac60b..d9932bf532 100644
--- a/lib/ansible/modules/crypto/openssl_csr.py
+++ b/lib/ansible/modules/crypto/openssl_csr.py
@@ -21,6 +21,9 @@ description:
     - It uses the pyOpenSSL python library to interact with openssl. This module supports
       the subjectAltName, keyUsage, extendedKeyUsage, basicConstraints and OCSP Must Staple
       extensions.
+    - "Please note that the module regenerates existing CSR if it doesn't match the module's
+      options, or if it seems to be corrupt. If you are concerned that this could overwrite
+      your existing CSR, consider using the I(backup) option."
 requirements:
     - Either cryptography >= 1.3
     - Or pyOpenSSL >= 0.15
@@ -190,6 +193,13 @@ options:
         default: auto
         choices: [ auto, cryptography, pyopenssl ]
         version_added: '2.8'
+    backup:
+        description:
+            - Create a backup file including a timestamp so you can get the original
+              CSR back if you overwrote it with a new one by accident.
+        type: bool
+        default: no
+        version_added: "2.8"
 extends_documentation_fragment:
 - files
 notes:
@@ -311,6 +321,11 @@ ocsp_must_staple:
     returned: changed or success
     type: bool
     sample: false
+backup_file:
+    description: Name of backup file created.
+    returned: changed and if I(backup) is C(yes)
+    type: str
+    sample: /path/to/www.ansible.com.csr.2019-03-09@11:22~
 '''
 
 import abc
@@ -394,6 +409,9 @@ class CertificateSigningRequestBase(crypto_utils.OpenSSLObject):
         self.request = None
         self.privatekey = None
 
+        self.backup = module.params['backup']
+        self.backup_file = None
+
         self.subject = [
             ('C', module.params['country_name']),
             ('ST', module.params['state_or_province_name']),
@@ -422,6 +440,8 @@ class CertificateSigningRequestBase(crypto_utils.OpenSSLObject):
         '''Generate the certificate signing request.'''
         if not self.check(module, perms_required=False) or self.force:
             result = self._generate_csr()
+            if self.backup:
+                self.backup_file = module.backup_local(self.path)
             crypto_utils.write_file(module, result)
             self.changed = True
 
@@ -448,6 +468,11 @@ class CertificateSigningRequestBase(crypto_utils.OpenSSLObject):
 
         return self._check_csr()
 
+    def remove(self, module):
+        if self.backup:
+            self.backup_file = module.backup_local(self.path)
+        super(CertificateSigningRequestBase, self).remove(module)
+
     def dump(self):
         '''Serialize the object into a dictionary.'''
 
@@ -462,6 +487,8 @@ class CertificateSigningRequestBase(crypto_utils.OpenSSLObject):
             'ocspMustStaple': self.ocspMustStaple,
             'changed': self.changed
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         return result
 
@@ -829,6 +856,7 @@ def main():
             basic_constraints_critical=dict(type='bool', default=False, aliases=['basicConstraints_critical']),
             ocsp_must_staple=dict(type='bool', default=False, aliases=['ocspMustStaple']),
             ocsp_must_staple_critical=dict(type='bool', default=False, aliases=['ocspMustStaple_critical']),
+            backup=dict(type='bool', default=False),
             select_crypto_backend=dict(type='str', default='auto', choices=['auto', 'cryptography', 'pyopenssl']),
         ),
         add_file_common_args=True,
diff --git a/lib/ansible/modules/crypto/openssl_dhparam.py b/lib/ansible/modules/crypto/openssl_dhparam.py
index 16c8a297e5..4cefb10823 100644
--- a/lib/ansible/modules/crypto/openssl_dhparam.py
+++ b/lib/ansible/modules/crypto/openssl_dhparam.py
@@ -19,6 +19,9 @@ short_description: Generate OpenSSL Diffie-Hellman Parameters
 description:
     - This module allows one to (re)generate OpenSSL DH-params.
     - This module uses file common arguments to specify generated file permissions.
+    - "Please note that the module regenerates existing DH params if they don't
+      match the module's options. If you are concerned that this could overwrite
+      your existing DH params, consider using the I(backup) option."
 requirements:
     - OpenSSL
 author:
@@ -46,6 +49,13 @@ options:
             - Name of the file in which the generated parameters will be saved.
         type: path
         required: true
+    backup:
+        description:
+            - Create a backup file including a timestamp so you can get the original
+              DH params back if you overwrote them with new ones by accident.
+        type: bool
+        default: no
+        version_added: "2.8"
 extends_documentation_fragment:
 - files
 seealso:
@@ -83,6 +93,11 @@ filename:
     returned: changed or success
     type: str
     sample: /etc/ssl/dhparams.pem
+backup_file:
+    description: Name of backup file created.
+    returned: changed and if I(backup) is C(yes)
+    type: str
+    sample: /path/to/dhparams.pem.2019-03-09@11:22~
 '''
 
 import os
@@ -107,6 +122,9 @@ class DHParameter(object):
         self.changed = False
         self.openssl_bin = module.get_bin_path('openssl', True)
 
+        self.backup = module.params['backup']
+        self.backup_file = None
+
     def generate(self, module):
         """Generate a keypair."""
         changed = False
@@ -122,6 +140,8 @@ class DHParameter(object):
             rc, dummy, err = module.run_command(command, check_rc=False)
             if rc != 0:
                 raise DHParameterError(to_native(err))
+            if self.backup:
+                self.backup_file = module.backup_local(self.path)
             try:
                 module.atomic_move(tmpsrc, self.path)
             except Exception as e:
@@ -137,6 +157,8 @@ class DHParameter(object):
         self.changed = changed
 
     def remove(self, module):
+        if self.backup:
+            self.backup_file = module.backup_local(self.path)
         try:
             os.remove(self.path)
             self.changed = True
@@ -186,6 +208,9 @@ class DHParameter(object):
             'filename': self.path,
             'changed': self.changed,
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
+
         return result
 
 
@@ -198,6 +223,7 @@ def main():
             size=dict(type='int', default=4096),
             force=dict(type='bool', default=False),
             path=dict(type='path', required=True),
+            backup=dict(type='bool', default=False),
         ),
         supports_check_mode=True,
         add_file_common_args=True,
diff --git a/lib/ansible/modules/crypto/openssl_pkcs12.py b/lib/ansible/modules/crypto/openssl_pkcs12.py
index f0a912d043..41ed703139 100644
--- a/lib/ansible/modules/crypto/openssl_pkcs12.py
+++ b/lib/ansible/modules/crypto/openssl_pkcs12.py
@@ -86,6 +86,13 @@ options:
         description:
             - PKCS#12 file path to parse.
         type: path
+    backup:
+        description:
+            - Create a backup file including a timestamp so you can get the original
+              output file back if you overwrote it with a new one by accident.
+        type: bool
+        default: no
+        version_added: "2.8"
 extends_documentation_fragment:
     - files
 seealso:
@@ -155,6 +162,11 @@ privatekey:
     returned: changed or success
     type: str
     sample: /etc/ssl/private/ansible.com.pem
+backup_file:
+    description: Name of backup file created.
+    returned: changed and if I(backup) is C(yes)
+    type: str
+    sample: /path/to/ansible.com.pem.2019-03-09@11:22~
 '''
 
 import stat
@@ -203,6 +215,9 @@ class Pkcs(crypto_utils.OpenSSLObject):
         if module.params['mode'] is None:
             module.params['mode'] = '0400'
 
+        self.backup = module.params['backup']
+        self.backup_file = None
+
     def check(self, module, perms_required=True):
         """Ensure the resource is in its desired state."""
 
@@ -232,6 +247,8 @@ class Pkcs(crypto_utils.OpenSSLObject):
         }
         if self.privatekey_path:
             result['privatekey_path'] = self.privatekey_path
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         return result
 
@@ -261,12 +278,19 @@ class Pkcs(crypto_utils.OpenSSLObject):
             except crypto_utils.OpenSSLBadPassphraseError as exc:
                 raise PkcsError(exc)
 
+        if self.backup:
+            self.backup_file = module.backup_local(self.path)
         crypto_utils.write_file(
             module,
             self.pkcs12.export(self.passphrase, self.iter_size, self.maciter_size),
             0o600
         )
 
+    def remove(self, module):
+        if self.backup:
+            self.backup_file = module.backup_local(self.path)
+        super(Pkcs, self).remove(module)
+
     def parse(self, module):
         """Read PKCS#12 file."""
 
@@ -301,6 +325,7 @@ def main():
         privatekey_path=dict(type='path'),
         state=dict(type='str', default='present', choices=['absent', 'present']),
         src=dict(type='path'),
+        backup=dict(type='bool', default=False),
     )
 
     required_if = [
diff --git a/lib/ansible/modules/crypto/openssl_privatekey.py b/lib/ansible/modules/crypto/openssl_privatekey.py
index c4eb51f417..494b9c90c6 100644
--- a/lib/ansible/modules/crypto/openssl_privatekey.py
+++ b/lib/ansible/modules/crypto/openssl_privatekey.py
@@ -26,7 +26,7 @@ description:
     - "Please note that the module regenerates private keys if they don't match
       the module's options. In particular, if you provide another passphrase
       (or specify none), change the keysize, etc., the private key will be
-      regenerated. If you are concerned that this could overwrite your private key,
+      regenerated. If you are concerned that this could **overwrite your private key**,
       consider using the I(backup) option."
     - The module can use the cryptography Python library, or the pyOpenSSL Python
       library. By default, it tries to detect which one is available. This can be
diff --git a/lib/ansible/modules/crypto/openssl_publickey.py b/lib/ansible/modules/crypto/openssl_publickey.py
index b8144e362d..b189dffaad 100644
--- a/lib/ansible/modules/crypto/openssl_publickey.py
+++ b/lib/ansible/modules/crypto/openssl_publickey.py
@@ -59,6 +59,13 @@ options:
             - The passphrase for the private key.
         type: str
         version_added: "2.4"
+    backup:
+        description:
+            - Create a backup file including a timestamp so you can get the original
+              public key back if you overwrote it with a different one by accident.
+        type: bool
+        default: no
+        version_added: "2.8"
 extends_documentation_fragment:
 - files
 seealso:
@@ -129,6 +136,11 @@ fingerprint:
       sha256: "41:ab:c7:cb:d5:5f:30:60:46:99:ac:d4:00:70:cf:a1:76:4f:24:5d:10:24:57:5d:51:6e:09:97:df:2f:de:c7"
       sha384: "85:39:50:4e:de:d9:19:33:40:70:ae:10:ab:59:24:19:51:c3:a2:e4:0b:1c:b1:6e:dd:b3:0c:d9:9e:6a:46:af:da:18:f8:ef:ae:2e:c0:9a:75:2c:9b:b3:0f:3a:5f:3d"
       sha512: "fd:ed:5e:39:48:5f:9f:fe:7f:25:06:3f:79:08:cd:ee:a5:e7:b3:3d:13:82:87:1f:84:e1:f5:c7:28:77:53:94:86:56:38:69:f0:d9:35:22:01:1e:a6:60:...:0f:9b"
+backup_file:
+    description: Name of backup file created.
+    returned: changed and if I(backup) is C(yes)
+    type: str
+    sample: /path/to/publickey.pem.2019-03-09@11:22~
 '''
 
 import os
@@ -169,6 +181,9 @@ class PublicKey(crypto_utils.OpenSSLObject):
         self.privatekey = None
         self.fingerprint = {}
 
+        self.backup = module.params['backup']
+        self.backup_file = None
+
     def generate(self, module):
         """Generate the public key."""
 
@@ -197,6 +212,8 @@ class PublicKey(crypto_utils.OpenSSLObject):
                     )
                     publickey_content = crypto.dump_publickey(crypto.FILETYPE_PEM, self.privatekey)
 
+                if self.backup:
+                    self.backup_file = module.backup_local(self.path)
                 crypto_utils.write_file(module, publickey_content)
 
                 self.changed = True
@@ -253,6 +270,11 @@ class PublicKey(crypto_utils.OpenSSLObject):
 
         return _check_privatekey()
 
+    def remove(self, module):
+        if self.backup:
+            self.backup_file = module.backup_local(self.path)
+        super(PublicKey, self).remove(module)
+
     def dump(self):
         """Serialize the object into a dictionary."""
 
@@ -263,6 +285,8 @@ class PublicKey(crypto_utils.OpenSSLObject):
             'changed': self.changed,
             'fingerprint': self.fingerprint,
         }
+        if self.backup_file:
+            result['backup_file'] = self.backup_file
 
         return result
 
@@ -277,6 +301,7 @@ def main():
             privatekey_path=dict(type='path'),
             format=dict(type='str', default='PEM', choices=['OpenSSH', 'PEM']),
             privatekey_passphrase=dict(type='str', no_log=True),
+            backup=dict(type='bool', default=False),
         ),
         supports_check_mode=True,
         add_file_common_args=True,
diff --git a/test/integration/targets/openssl_certificate/tasks/ownca.yml b/test/integration/targets/openssl_certificate/tasks/ownca.yml
index 59f544a3d4..acdc2a437c 100644
--- a/test/integration/targets/openssl_certificate/tasks/ownca.yml
+++ b/test/integration/targets/openssl_certificate/tasks/ownca.yml
@@ -217,4 +217,54 @@
     ownca_digest: sha256
   register: ownca_broken
 
+- name: (OwnCA, {{select_crypto_backend}}) Backup test
+  openssl_certificate:
+    path: '{{ output_dir }}/ownca_cert_backup.pem'
+    csr_path: '{{ output_dir }}/csr_ecc.csr'
+    ownca_path: '{{ output_dir }}/ca_cert.pem'
+    ownca_privatekey_path: '{{ output_dir }}/privatekey.pem'
+    provider: ownca
+    ownca_digest: sha256
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: ownca_backup_1
+- name: (OwnCA, {{select_crypto_backend}}) Backup test (idempotent)
+  openssl_certificate:
+    path: '{{ output_dir }}/ownca_cert_backup.pem'
+    csr_path: '{{ output_dir }}/csr_ecc.csr'
+    ownca_path: '{{ output_dir }}/ca_cert.pem'
+    ownca_privatekey_path: '{{ output_dir }}/privatekey.pem'
+    provider: ownca
+    ownca_digest: sha256
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: ownca_backup_2
+- name: (OwnCA, {{select_crypto_backend}}) Backup test (change)
+  openssl_certificate:
+    path: '{{ output_dir }}/ownca_cert_backup.pem'
+    csr_path: '{{ output_dir }}/csr.csr'
+    ownca_path: '{{ output_dir }}/ca_cert.pem'
+    ownca_privatekey_path: '{{ output_dir }}/privatekey.pem'
+    provider: ownca
+    ownca_digest: sha256
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: ownca_backup_3
+- name: (OwnCA, {{select_crypto_backend}}) Backup test (remove)
+  openssl_certificate:
+    path: '{{ output_dir }}/ownca_cert_backup.pem'
+    state: absent
+    provider: ownca
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: ownca_backup_4
+- name: (OwnCA, {{select_crypto_backend}}) Backup test (remove, idempotent)
+  openssl_certificate:
+    path: '{{ output_dir }}/ownca_cert_backup.pem'
+    state: absent
+    provider: ownca
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: ownca_backup_5
+
 - import_tasks: ../tests/validate_ownca.yml
diff --git a/test/integration/targets/openssl_certificate/tasks/selfsigned.yml b/test/integration/targets/openssl_certificate/tasks/selfsigned.yml
index 06c73ded9c..2398639a45 100644
--- a/test/integration/targets/openssl_certificate/tasks/selfsigned.yml
+++ b/test/integration/targets/openssl_certificate/tasks/selfsigned.yml
@@ -224,4 +224,51 @@
     selfsigned_digest: sha256
   register: selfsigned_broken
 
+- name: (Selfsigned, {{select_crypto_backend}}) Backup test
+  openssl_certificate:
+    path: '{{ output_dir }}/selfsigned_cert_backup.pem'
+    csr_path: '{{ output_dir }}/csr_ecc.csr'
+    privatekey_path: '{{ output_dir }}/privatekey_ecc.pem'
+    provider: selfsigned
+    selfsigned_digest: sha256
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: selfsigned_backup_1
+- name: (Selfsigned, {{select_crypto_backend}}) Backup test (idempotent)
+  openssl_certificate:
+    path: '{{ output_dir }}/selfsigned_cert_backup.pem'
+    csr_path: '{{ output_dir }}/csr_ecc.csr'
+    privatekey_path: '{{ output_dir }}/privatekey_ecc.pem'
+    provider: selfsigned
+    selfsigned_digest: sha256
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: selfsigned_backup_2
+- name: (Selfsigned, {{select_crypto_backend}}) Backup test (change)
+  openssl_certificate:
+    path: '{{ output_dir }}/selfsigned_cert_backup.pem'
+    csr_path: '{{ output_dir }}/csr.csr'
+    privatekey_path: '{{ output_dir }}/privatekey.pem'
+    provider: selfsigned
+    selfsigned_digest: sha256
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: selfsigned_backup_3
+- name: (Selfsigned, {{select_crypto_backend}}) Backup test (remove)
+  openssl_certificate:
+    path: '{{ output_dir }}/selfsigned_cert_backup.pem'
+    state: absent
+    provider: selfsigned
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: selfsigned_backup_4
+- name: (Selfsigned, {{select_crypto_backend}}) Backup test (remove, idempotent)
+  openssl_certificate:
+    path: '{{ output_dir }}/selfsigned_cert_backup.pem'
+    state: absent
+    provider: selfsigned
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: selfsigned_backup_5
+
 - import_tasks: ../tests/validate_selfsigned.yml
diff --git a/test/integration/targets/openssl_certificate/tests/validate_ownca.yml b/test/integration/targets/openssl_certificate/tests/validate_ownca.yml
index b3f07ba352..3d90d75fcc 100644
--- a/test/integration/targets/openssl_certificate/tests/validate_ownca.yml
+++ b/test/integration/targets/openssl_certificate/tests/validate_ownca.yml
@@ -106,3 +106,17 @@
   assert:
     that:
       - ownca_broken is changed
+
+- name: Check backup
+  assert:
+    that:
+      - ownca_backup_1 is changed
+      - ownca_backup_1.backup_file is undefined
+      - ownca_backup_2 is not changed
+      - ownca_backup_2.backup_file is undefined
+      - ownca_backup_3 is changed
+      - ownca_backup_3.backup_file is string
+      - ownca_backup_4 is changed
+      - ownca_backup_4.backup_file is string
+      - ownca_backup_5 is not changed
+      - ownca_backup_5.backup_file is undefined
diff --git a/test/integration/targets/openssl_certificate/tests/validate_selfsigned.yml b/test/integration/targets/openssl_certificate/tests/validate_selfsigned.yml
index b77b44885d..a357f7f816 100644
--- a/test/integration/targets/openssl_certificate/tests/validate_selfsigned.yml
+++ b/test/integration/targets/openssl_certificate/tests/validate_selfsigned.yml
@@ -107,3 +107,17 @@
   assert:
     that:
       - selfsigned_broken is changed
+
+- name: Check backup
+  assert:
+    that:
+      - selfsigned_backup_1 is changed
+      - selfsigned_backup_1.backup_file is undefined
+      - selfsigned_backup_2 is not changed
+      - selfsigned_backup_2.backup_file is undefined
+      - selfsigned_backup_3 is changed
+      - selfsigned_backup_3.backup_file is string
+      - selfsigned_backup_4 is changed
+      - selfsigned_backup_4.backup_file is string
+      - selfsigned_backup_5 is not changed
+      - selfsigned_backup_5.backup_file is undefined
diff --git a/test/integration/targets/openssl_csr/tasks/impl.yml b/test/integration/targets/openssl_csr/tasks/impl.yml
index d5861d8635..f5af14cfa7 100644
--- a/test/integration/targets/openssl_csr/tasks/impl.yml
+++ b/test/integration/targets/openssl_csr/tasks/impl.yml
@@ -288,3 +288,45 @@
     useCommonNameForSAN: no
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: output_broken
+
+- name: Generate CSR
+  openssl_csr:
+    path: '{{ output_dir }}/csr_backup.csr'
+    privatekey_path: '{{ output_dir }}/privatekey.pem'
+    subject:
+      commonName: www.ansible.com
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: csr_backup_1
+- name: Generate CSR (idempotent)
+  openssl_csr:
+    path: '{{ output_dir }}/csr_backup.csr'
+    privatekey_path: '{{ output_dir }}/privatekey.pem'
+    subject:
+      commonName: www.ansible.com
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: csr_backup_2
+- name: Generate CSR (change)
+  openssl_csr:
+    path: '{{ output_dir }}/csr_backup.csr'
+    privatekey_path: '{{ output_dir }}/privatekey.pem'
+    subject:
+      commonName: ansible.com
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: csr_backup_3
+- name: Generate CSR (remove)
+  openssl_csr:
+    path: '{{ output_dir }}/csr_backup.csr'
+    state: absent
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: csr_backup_4
+- name: Generate CSR (remove, idempotent)
+  openssl_csr:
+    path: '{{ output_dir }}/csr_backup.csr'
+    state: absent
+    backup: yes
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: csr_backup_5
diff --git a/test/integration/targets/openssl_csr/tests/validate.yml b/test/integration/targets/openssl_csr/tests/validate.yml
index a6d36a8d07..585cf64c0a 100644
--- a/test/integration/targets/openssl_csr/tests/validate.yml
+++ b/test/integration/targets/openssl_csr/tests/validate.yml
@@ -124,3 +124,17 @@
   assert:
     that:
       - output_broken is changed
+
+- name: Check backup
+  assert:
+    that:
+      - csr_backup_1 is changed
+      - csr_backup_1.backup_file is undefined
+      - csr_backup_2 is not changed
+      - csr_backup_2.backup_file is undefined
+      - csr_backup_3 is changed
+      - csr_backup_3.backup_file is string
+      - csr_backup_4 is changed
+      - csr_backup_4.backup_file is string
+      - csr_backup_5 is not changed
+      - csr_backup_5.backup_file is undefined
diff --git a/test/integration/targets/openssl_dhparam/tasks/main.yml b/test/integration/targets/openssl_dhparam/tasks/main.yml
index 7612ec8932..bf788abbd8 100644
--- a/test/integration/targets/openssl_dhparam/tasks/main.yml
+++ b/test/integration/targets/openssl_dhparam/tasks/main.yml
@@ -53,4 +53,36 @@
         force: yes
       register: output_broken
 
+    - name: Generate params
+      openssl_dhparam:
+        path: '{{ output_dir }}/dh_backup.pem'
+        size: 512
+        backup: yes
+      register: dhparam_backup_1
+    - name: Generate params (idempotent)
+      openssl_dhparam:
+        path: '{{ output_dir }}/dh_backup.pem'
+        size: 512
+        backup: yes
+      register: dhparam_backup_2
+    - name: Generate params (change)
+      openssl_dhparam:
+        path: '{{ output_dir }}/dh_backup.pem'
+        size: 512
+        force: yes
+        backup: yes
+      register: dhparam_backup_3
+    - name: Generate params (remove)
+      openssl_dhparam:
+        path: '{{ output_dir }}/dh_backup.pem'
+        state: absent
+        backup: yes
+      register: dhparam_backup_4
+    - name: Generate params (remove, idempotent)
+      openssl_dhparam:
+        path: '{{ output_dir }}/dh_backup.pem'
+        state: absent
+        backup: yes
+      register: dhparam_backup_5
+
     - import_tasks: ../tests/validate.yml
diff --git a/test/integration/targets/openssl_dhparam/tests/validate.yml b/test/integration/targets/openssl_dhparam/tests/validate.yml
index 380cd2cba5..b4a2857255 100644
--- a/test/integration/targets/openssl_dhparam/tests/validate.yml
+++ b/test/integration/targets/openssl_dhparam/tests/validate.yml
@@ -35,3 +35,17 @@
   assert:
     that:
       - output_broken is changed
+
+- name: Check backup
+  assert:
+    that:
+      - dhparam_backup_1 is changed
+      - dhparam_backup_1.backup_file is undefined
+      - dhparam_backup_2 is not changed
+      - dhparam_backup_2.backup_file is undefined
+      - dhparam_backup_3 is changed
+      - dhparam_backup_3.backup_file is string
+      - dhparam_backup_4 is changed
+      - dhparam_backup_4.backup_file is string
+      - dhparam_backup_5 is not changed
+      - dhparam_backup_5.backup_file is undefined
diff --git a/test/integration/targets/openssl_pkcs12/tasks/impl.yml b/test/integration/targets/openssl_pkcs12/tasks/impl.yml
index f463401333..5431861002 100644
--- a/test/integration/targets/openssl_pkcs12/tasks/impl.yml
+++ b/test/integration/targets/openssl_pkcs12/tasks/impl.yml
@@ -1,3 +1,4 @@
+---
 - block:
   - name: 'Generate privatekey with'
     openssl_privatekey:
@@ -115,6 +116,47 @@
       mode: 0644
     register: output_broken
 
+  - name: 'Generate PKCS#12 file'
+    openssl_pkcs12:
+      path: "{{ output_dir }}/ansible_backup.p12"
+      friendly_name: 'abracadabra'
+      privatekey_path: "{{ output_dir }}/ansible_pkey.pem"
+      certificate_path: "{{ output_dir }}/ansible.crt"
+      state: present
+      backup: yes
+    register: p12_backup_1
+  - name: 'Generate PKCS#12 file (idempotent)'
+    openssl_pkcs12:
+      path: "{{ output_dir }}/ansible_backup.p12"
+      friendly_name: 'abracadabra'
+      privatekey_path: "{{ output_dir }}/ansible_pkey.pem"
+      certificate_path: "{{ output_dir }}/ansible.crt"
+      state: present
+      backup: yes
+    register: p12_backup_2
+  - name: 'Generate PKCS#12 file (change)'
+    openssl_pkcs12:
+      path: "{{ output_dir }}/ansible_backup.p12"
+      friendly_name: 'abra'
+      privatekey_path: "{{ output_dir }}/ansible_pkey.pem"
+      certificate_path: "{{ output_dir }}/ansible.crt"
+      state: present
+      force: yes  # FIXME: idempotency does not work, so we have to force! (https://github.com/ansible/ansible/issues/53221)
+      backup: yes
+    register: p12_backup_3
+  - name: 'Generate PKCS#12 file (remove)'
+    openssl_pkcs12:
+      path: "{{ output_dir }}/ansible_backup.p12"
+      state: absent
+      backup: yes
+    register: p12_backup_4
+  - name: 'Generate PKCS#12 file (remove, idempotent)'
+    openssl_pkcs12:
+      path: "{{ output_dir }}/ansible_backup.p12"
+      state: absent
+      backup: yes
+    register: p12_backup_5
+
   - import_tasks: ../tests/validate.yml
 
   always:
diff --git a/test/integration/targets/openssl_pkcs12/tests/validate.yml b/test/integration/targets/openssl_pkcs12/tests/validate.yml
index a44622d8a0..da88d0c7b7 100644
--- a/test/integration/targets/openssl_pkcs12/tests/validate.yml
+++ b/test/integration/targets/openssl_pkcs12/tests/validate.yml
@@ -22,7 +22,7 @@
       - p12_force.changed
       - p12_force_and_mode.mode == '0644' and p12_force_and_mode.changed
 
-- name:
+- name: Check passphrase on private key
   assert:
     that:
       - passphrase_error_1 is failed
@@ -36,3 +36,17 @@
   assert:
     that:
       - output_broken is changed
+
+- name: Check backup
+  assert:
+    that:
+      - p12_backup_1 is changed
+      - p12_backup_1.backup_file is undefined
+      - p12_backup_2 is not changed
+      - p12_backup_2.backup_file is undefined
+      - p12_backup_3 is changed
+      - p12_backup_3.backup_file is string
+      - p12_backup_4 is changed
+      - p12_backup_4.backup_file is string
+      - p12_backup_5 is not changed
+      - p12_backup_5.backup_file is undefined
diff --git a/test/integration/targets/openssl_publickey/tasks/main.yml b/test/integration/targets/openssl_publickey/tasks/main.yml
index 9ab1c24487..c83c3971cb 100644
--- a/test/integration/targets/openssl_publickey/tasks/main.yml
+++ b/test/integration/targets/openssl_publickey/tasks/main.yml
@@ -74,9 +74,23 @@
         curve: secp256k1
 
     - name: Generate publickey 5 - PEM format
+      openssl_publickey:
+        path: '{{ output_dir }}/publickey5.pub'
+        privatekey_path: '{{ output_dir }}/privatekey.pem'
+        backup: yes
+      register: privatekey5_1
+    - name: Generate publickey 5 - PEM format (idempotent)
+      openssl_publickey:
+        path: '{{ output_dir }}/publickey5.pub'
+        privatekey_path: '{{ output_dir }}/privatekey.pem'
+        backup: yes
+      register: privatekey5_2
+    - name: Generate publickey 5 - PEM format (different private key)
       openssl_publickey:
         path: '{{ output_dir }}/publickey5.pub'
         privatekey_path: '{{ output_dir }}/privatekey5.pem'
+        backup: yes
+      register: privatekey5_3
 
     - name: Generate privatekey with password
       openssl_privatekey:
@@ -118,6 +132,25 @@
         privatekey_path: '{{ output_dir }}/privatekey5.pem'
       register: output_broken
 
+    - name: Generate publickey - PEM format (for removal)
+      openssl_publickey:
+        path: '{{ output_dir }}/publickey_removal.pub'
+        privatekey_path: '{{ output_dir }}/privatekey.pem'
+    - name: Generate publickey - PEM format (removal)
+      openssl_publickey:
+        state: absent
+        path: '{{ output_dir }}/publickey_removal.pub'
+        privatekey_path: '{{ output_dir }}/privatekey.pem'
+        backup: yes
+      register: remove_1
+    - name: Generate publickey - PEM format (removal, idempotent)
+      openssl_publickey:
+        state: absent
+        path: '{{ output_dir }}/publickey_removal.pub'
+        privatekey_path: '{{ output_dir }}/privatekey.pem'
+        backup: yes
+      register: remove_2
+
     - import_tasks: ../tests/validate.yml
 
   when: pyopenssl_version.stdout is version('16.0.0', '>=')
diff --git a/test/integration/targets/openssl_publickey/tests/validate.yml b/test/integration/targets/openssl_publickey/tests/validate.yml
index 2a012884fd..83011b210c 100644
--- a/test/integration/targets/openssl_publickey/tests/validate.yml
+++ b/test/integration/targets/openssl_publickey/tests/validate.yml
@@ -83,6 +83,16 @@
       - publickey4_modulus.stdout == privatekey4_modulus.stdout
   when: openssl_version.stdout is version('0.9.8zh', '>=')
 
+- name: Validate idempotency and backup
+  assert:
+    that:
+      - privatekey5_1 is changed
+      - privatekey5_1.backup_file is undefined
+      - privatekey5_2 is not changed
+      - privatekey5_2.backup_file is undefined
+      - privatekey5_3 is changed
+      - privatekey5_3.backup_file is string
+
 - name: Validate public key 5 (test - privatekey's pubkey)
   command: 'openssl ec -in {{ output_dir }}/privatekey5.pem -pubout'
   register: privatekey5_pubkey
@@ -111,3 +121,11 @@
   assert:
     that:
       - output_broken is changed
+
+- name: Validate remove
+  assert:
+    that:
+      - remove_1 is changed
+      - remove_2 is not changed
+      - remove_1.backup_file is string
+      - remove_2.backup_file is undefined