diff --git a/changelogs/fragments/6199-archive-generate-checksum-in-chunks.yml b/changelogs/fragments/6199-archive-generate-checksum-in-chunks.yml new file mode 100644 index 0000000000..ed0a1e2dca --- /dev/null +++ b/changelogs/fragments/6199-archive-generate-checksum-in-chunks.yml @@ -0,0 +1,2 @@ +bugfixes: + - archive - reduce RAM usage by generating CRC32 checksum over chunks (https://github.com/ansible-collections/community.general/pull/6274). diff --git a/plugins/modules/files/archive.py b/plugins/modules/files/archive.py index 5aaab4e0d0..882240c089 100644 --- a/plugins/modules/files/archive.py +++ b/plugins/modules/files/archive.py @@ -601,7 +601,13 @@ class TarArchive(Archive): # The python implementations of gzip, bz2, and lzma do not support restoring compressed files # to their original names so only file checksum is returned f = self._open_compressed_file(_to_native_ascii(path), 'r') - checksums = set([(b'', crc32(f.read()))]) + checksum = 0 + while True: + chunk = f.read(16 * 1024 * 1024) + if not chunk: + break + checksum = crc32(chunk, checksum) + checksums = set([(b'', checksum)]) f.close() except Exception: checksums = set()