From 00b3468fd94337efd2e6dbd4a45f67e04c682c41 Mon Sep 17 00:00:00 2001 From: "pierre.debris" Date: Wed, 3 Jul 2024 17:52:40 +0200 Subject: [PATCH] feat[redis cache plugin]: Add Redis connection parameters full control support --- ...dis-connection-parameters-full-control.yml | 2 + plugins/cache/redis.py | 271 +++++++++++++++--- 2 files changed, 233 insertions(+), 40 deletions(-) create mode 100644 changelogs/fragments/8578-feat-add-redis-connection-parameters-full-control.yml diff --git a/changelogs/fragments/8578-feat-add-redis-connection-parameters-full-control.yml b/changelogs/fragments/8578-feat-add-redis-connection-parameters-full-control.yml new file mode 100644 index 0000000000..f407b97195 --- /dev/null +++ b/changelogs/fragments/8578-feat-add-redis-connection-parameters-full-control.yml @@ -0,0 +1,2 @@ +minor_changes: + - redis cache plugin - add Redis connection parameters full access in order to be able to set advanced socket options, like enabling keep alive (https://github.com/ansible-collections/community.general/pull/8578) \ No newline at end of file diff --git a/plugins/cache/redis.py b/plugins/cache/redis.py index e01083e863..f942c04dcc 100644 --- a/plugins/cache/redis.py +++ b/plugins/cache/redis.py @@ -15,6 +15,165 @@ DOCUMENTATION = ''' requirements: - redis>=2.4.5 (python lib) options: + _decode_responses: + description: If set to `true`, returned values from Redis commands get decoded automatically using the client's charset value. + type: bool + default: false + env: + - name: ANSIBLE_CACHE_REDIS_DECODE_RESPONSES + ini: + - key: fact_caching_redis_decode_responses + section: defaults + _encoding: + description: Set the charset to use for facts encoding. + type: string + default: utf-8 + env: + - name: ANSIBLE_CACHE_REDIS_ENCODING + ini: + - key: fact_caching_redis_encoding + section: defaults + _encoding_errors: + description: + - The error handling scheme to use for encoding errors. + - The default is `strict` meaning that encoding errors raise a `UnicodeEncodeError`. + - See https://docs.python.org/fr/3/library/stdtypes.html#str.encode for more details. + type: string + default: strict + choices: [ backslashreplace, ignore, replace, strict, xmlcharrefreplace ] + env: + - name: ANSIBLE_CACHE_REDIS_ENCODING_ERRORS + ini: + - key: fact_caching_redis_encoding_errors + section: defaults + _keyset_name: + description: User defined name for cache keyset name. + type: string + default: ansible_cache_keys + env: + - name: ANSIBLE_CACHE_REDIS_KEYSET_NAME + ini: + - key: fact_caching_redis_keyset_name + section: defaults + version_added: 1.3.0 + _prefix: + description: User defined prefix to use when creating the DB entries + type: string + default: ansible_facts + env: + - name: ANSIBLE_CACHE_PLUGIN_PREFIX + ini: + - key: fact_caching_prefix + section: defaults + _retry_on_timeout: + description: + - Controls how socket.timeout errors are handled. + - When set to `false` a TimeoutError will be raised anytime a socket.timeout is encountered. + - When set to `true`, it enable retries like other `socket.error`s. + type: bool + default: false + env: + - name: ANSIBLE_CACHE_REDIS_RETRY_ON_TIMEOUT + ini: + - key: fact_caching_redis_retry_on_timeout + section: defaults + _sentinel_service_name: + description: The redis sentinel service name (or referenced as cluster name). + type: string + env: + - name: ANSIBLE_CACHE_REDIS_SENTINEL + ini: + - key: fact_caching_redis_sentinel + section: defaults + version_added: 1.3.0 + _socket_connect_timeout: + description: + - Timeout value, in seconds, for Redis socket connection. + - If not set, connection timeout is disabled. + type: integer + env: + - name: ANSIBLE_CACHE_REDIS_SOCKET_CONNECT_TIMEOUT + ini: + - key: fact_caching_redis_socket_connect_timeout + section: defaults + _socket_keepalive: + description: + - Specifies whether to enable keepalive for Redis socket connection. + type: bool + default: false + env: + - name: ANSIBLE_CACHE_REDIS_SOCKET_KEEPALIVE + ini: + - key: fact_caching_redis_socket_keepalive + section: defaults + _socket_keepalive_options: + description: + - Finer grain control keepalive options when `_socket_keepalive` is set to `true`. + - A comma separated socket options string of : pairs, for example V(TCP_KEEPIDLE:600,TCP_KEEPCNT=10,TCP_KEEPINTVL:300). + - Accepted keys are `TCP_KEEPIDLE`, `TCP_KEEPCNT`, and `TCP_KEEPINTVL`. + - Integers are expected for values. + type: string + env: + - name: ANSIBLE_CACHE_REDIS_SOCKET_KEEPALIVE_OPTIONS + ini: + - key: fact_caching_redis_socket_keepalive_options + section: defaults + _socket_timeout: + description: + - Timeout value, in seconds, for Redis socket connection. + - If not set, timeout is disabled. + type: integer + env: + - name: ANSIBLE_CACHE_REDIS_SOCKET_TIMEOUT + ini: + - key: fact_caching_redis_socket_timeout + section: defaults + _ssl_ca_certs_file: + description: When using SSL on Redis connection, specifies the SSL CA file path. + type: string + env: + - name: ANSIBLE_CACHE_REDIS_SSL_CA_CERTS_FILE + ini: + - key: fact_caching_redis_ssl_ca_certs_file + section: defaults + _ssl_cert_file: + description: When using SSL on Redis connection, specifies the SSL certificate file path. + type: string + env: + - name: ANSIBLE_CACHE_REDIS_SSL_CERT_FILE + ini: + - key: fact_caching_redis_ssl_cert_file + section: defaults + _ssl_cert_reqs: + default: none + type: string + description: + - When using SSL on Redis connection, specifies the security mode to use. + - See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.verify_mode for more details. + env: + - name: ANSIBLE_CACHE_REDIS_SSL_CERT_REQ + ini: + - key: fact_caching_redis_ssl_cert_req + section: defaults + choices: [ none, optionnal, required ] + _ssl_key_file: + description: When using SSL on Redis connection, specifies the SSL key file path. + type: string + env: + - name: ANSIBLE_CACHE_REDIS_SSL_KEY_FILE + ini: + - key: fact_caching_redis_ssl_key_file + section: defaults + _timeout: + default: 86400 + description: Expiration timeout in seconds for the cache plugin data. Set to 0 to never expire + type: integer + # TODO: determine whether it is OK to change to: type: float + env: + - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT + ini: + - key: fact_caching_timeout + section: defaults _uri: description: - A colon separated string of connection information for Redis. @@ -28,44 +187,6 @@ DOCUMENTATION = ''' ini: - key: fact_caching_connection section: defaults - _prefix: - description: User defined prefix to use when creating the DB entries - type: string - default: ansible_facts - env: - - name: ANSIBLE_CACHE_PLUGIN_PREFIX - ini: - - key: fact_caching_prefix - section: defaults - _keyset_name: - description: User defined name for cache keyset name. - type: string - default: ansible_cache_keys - env: - - name: ANSIBLE_CACHE_REDIS_KEYSET_NAME - ini: - - key: fact_caching_redis_keyset_name - section: defaults - version_added: 1.3.0 - _sentinel_service_name: - description: The redis sentinel service name (or referenced as cluster name). - type: string - env: - - name: ANSIBLE_CACHE_REDIS_SENTINEL - ini: - - key: fact_caching_redis_sentinel - section: defaults - version_added: 1.3.0 - _timeout: - default: 86400 - type: integer - # TODO: determine whether it is OK to change to: type: float - description: Expiration timeout in seconds for the cache plugin data. Set to 0 to never expire - env: - - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT - ini: - - key: fact_caching_timeout - section: defaults ''' import re @@ -97,8 +218,11 @@ class CacheModule(BaseCacheModule): performance. """ _sentinel_service_name = None + _encoding_errors_choices = ['backslashreplace', 'ignore', 'replace', 'strict', 'xmlcharrefreplace'] + _socket_keepalive_available_opts = ['TCP_KEEPIDLE', 'TCP_KEEPCNT', 'TCP_KEEPINTVL'] re_url_conn = re.compile(r'^([^:]+|\[[^]]+\]):(\d+):(\d+)(?::(.*))?$') re_sent_conn = re.compile(r'^(.*):(\d+)$') + re_socket_keepalive_opts = re.compile(r'^(\w+:\d+)(?:,(\w+:\d+))+$') def __init__(self, *args, **kwargs): uri = '' @@ -110,17 +234,71 @@ class CacheModule(BaseCacheModule): self._prefix = self.get_option('_prefix') self._keys_set = self.get_option('_keyset_name') self._sentinel_service_name = self.get_option('_sentinel_service_name') + self._decode_responses = bool(self.get_option('_decode_responses')) + self._encoding = self.get_option('_encoding') + self._encoding_errors = self.get_option('_encoding_errors') + self._retry_on_timeout = bool(self.get_option('_retry_on_timeout')) + self._socket_keepalive = self.get_option('_socket_keepalive') + self._socket_connect_timeout = int(self.get_option('_socket_connect_timeout')) \ + if self._socket_keepalive and self.get_option('_socket_connect_timeout') else None + self._socket_keepalive_options = self._parse_socket_options(self.get_option('_socket_keepalive_options')) \ + if self._socket_keepalive and self.get_option('_socket_keepalive_options') else None + self._socket_timeout = int(self.get_option('_socket_timeout')) \ + if self._socket_keepalive and self.get_option('_socket_timeout') else None + self._ssl_ca_certs_file = self.get_option('_ssl_ca_certs_file') + self._ssl_cert_file = self.get_option('_ssl_cert_file') + self._ssl_cert_reqs = self.get_option('_ssl_cert_reqs') + self._ssl_key_file = self.get_option('_ssl_key_file') if not HAS_REDIS: raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'") + if self._encoding_errors not in self._encoding_errors_choices: + raise AnsibleError("Unsupported value '%s' for Redis cache plugin parameter '_encoding_errors'" % (self._encoding_errors)) + self._cache = {} - kw = {} + kw = { + 'decode_responses': self._decode_responses, + 'encoding_errors': self._encoding_errors, + 'encoding': self._encoding, + 'retry_on_timeout': self._retry_on_timeout, + 'socket_connect_timeout': self._socket_connect_timeout, + 'socket_keepalive_options': self._socket_keepalive_options, + 'socket_keepalive': self._socket_keepalive, + 'socket_timeout': self._socket_timeout, + } # tls connection tlsprefix = 'tls://' if uri.startswith(tlsprefix): - kw['ssl'] = True + from os import access, R_OK + from os.path import isfile + import ssl + + if not self._ssl_cert_reqs.upper() in list(map(lambda x: x.name.split('_')[1], ssl.VerifyMode)): + raise AnsibleError("Unsupported value '%s' for Redis cache plugin parameter '_ssl_cert_reqs'" % (self._ssl_cert_reqs)) + + if self._ssl_ca_certs_file: + if not isfile(self._ssl_ca_certs_file) and not access(self._ssl_ca_certs_file, R_OK): + raise AnsibleError("File %s doesn't exist or isn't readable for Redis cache plugin parameter '_ssl_ca_certs_file'" % + self._ssl_ca_certs_file) + + if self._ssl_cert_file: + if not isfile(self._ssl_cert_file) and not access(self._ssl_cert_file, R_OK): + raise AnsibleError("File %s doesn't exist or isn't readable for Redis cache plugin parameter '_ssl_cert_file'" % self._ssl_cert_file) + + if self._ssl_key_file: + if not isfile(self._ssl_key_file) and not access(self._ssl_key_file, R_OK): + raise AnsibleError("File %s doesn't exist or isn't readable for Redis cache plugin parameter '_ssl_key_file'" % self._ssl_key_file) + + kw.update({ + 'ssl': True, + 'ssl_keyfile': self._ssl_key_file, + 'ssl_certfile': self._ssl_cert_file, + 'ssl_cert_reqs': self._ssl_cert_reqs, + 'ssl_ca_certs': self._ssl_ca_certs_file + }) + uri = uri[len(tlsprefix):] # redis sentinel connection @@ -132,6 +310,7 @@ class CacheModule(BaseCacheModule): self._db = StrictRedis(*connection, **kw) display.vv('Redis connection: %s' % self._db) + display.vvv("Redis connection kwargs: %s" % ({**self._db.get_connection_kwargs(), **{'password': '****'}})) @staticmethod def _parse_connection(re_patt, uri): @@ -140,6 +319,18 @@ class CacheModule(BaseCacheModule): raise AnsibleError("Unable to parse connection string") return match.groups() + def _parse_socket_options(self, options): + if not self.re_socket_keepalive_opts.match(options): + raise AnsibleError("Unable to parse Redis cache socket keepalive options string") + import socket + opts = {} + for opt in options.split(','): + key, value = opt.split(':') + if key not in self._socket_keepalive_available_opts: + raise AnsibleError("Option '%s' is not available for parameter '_socket_keepalive_options' for Redis cache plugin" % (key)) + opts[getattr(socket, key)] = int(value) + return opts + def _get_sentinel_connection(self, uri, kw): """ get sentinel connection details from _uri