From d5ae6cc5857dc65aad49e2afcfcf099ff1ae1f0d Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Mon, 10 Jul 2017 09:05:42 +0200 Subject: [PATCH] postgres_db: add dump and restore support (#20627) * Feature #2731: added postgres import and dump * Feature #2731: be more permissive of arguments ``` hacking/test-module -m ./ppostgresql_db.py -a "db=example state=dump target=/tmp/out"` ``` failed previously since host, user, and port were required as keywords in the pg_dump / pg_import methods. * Feature #2731: fixed doc string for validate-modules ``` $ ansible-validate-modules database/postgresql/ ``` now passes. * Feature #2731: disable 'password' for dump/restore * Feature #2731: bump added version to 2.3 * Feature #2731: replace db_import with db_restore * Feature #2731: add missing version description * Feature #2731: fix 'state' description * Feature #2731: fix pep8 issues * Feature #2731: put state documentation in a single string * Bump added version from 2.3 to 2.4 * Fix pep8 and pylint errors * Attempt yaml formatting of documentation string * Add integration tests for postgres_db:dump/restore * Update dump/restore logic to support new kw-args Also attempt to support password; integration tests are still failing. * Revert to postgres user for dump/restore Passing PGPASSWORD is not working for subprocesses. For the moment, reverting to the strategy of failing if login_password is set and using `postgres` for all testing of dump/restore. * Various cleanups to have tests passing * Working tests for {sql,tar} x {,bz2,gz,xz} * Use pg_user to support FreeBSD * Revert login_ prefixes and re-enable password support All `login_` keywords are mapped to their non-prefix versions so the previous changes were effectively using `postgres` for all actions. With the proper keywords, PGPASSWORD-passing to the subprocess is now working. * Optionally add password environ_update doesn't handle None values in the dictionary to be added to the environment. Adding check. * Quick fixes * Refactor login arguments after fixes from pchauncey The fixes introduced by pchaunchy pointed to further issues (like no --dbname on PG<=9.2) with the login parameters. This refactors them and adds further tests. Note: this will still not pass integration tests due to a further issue with pg_dump as a non-admin user: pg_restore: [archiver (db)] Error while PROCESSING TOC: pg_restore: [archiver (db)] Error from TOC entry 1925; 0 0 COMMENT EXTENSION plpgsql pg_restore: [archiver (db)] could not execute query: ERROR: must be owner of extension plpgsql * Introduce target_opts for passing limiting dumped/restored schemas The current integration tests (PG version and template DBs) don't permit a regular user (`{{ db_user1 }}`) access to plpgsql causing restores to fail. By adding an option for passing arbitrary args to pg_dump and pg_restore, testing is made easier. This also paves the way for `-j` usage, once the PG version is bumped. --- .../database/postgresql/postgresql_db.py | 186 +++++++++++++++++- .../targets/postgresql/tasks/main.yml | 15 ++ .../postgresql/tasks/state_dump_restore.yml | 113 +++++++++++ 3 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 test/integration/targets/postgresql/tasks/state_dump_restore.yml diff --git a/lib/ansible/modules/database/postgresql/postgresql_db.py b/lib/ansible/modules/database/postgresql/postgresql_db.py index 844bc4cceb..f15510c5be 100644 --- a/lib/ansible/modules/database/postgresql/postgresql_db.py +++ b/lib/ansible/modules/database/postgresql/postgresql_db.py @@ -61,11 +61,25 @@ options: required: false default: null state: - description: - - The database state + description: | + The database state. present implies that the database should be created if necessary. + absent implies that the database should be removed if present. + dump requires a target definition to which the database will be backed up. + (Added in 2.4) restore also requires a target definition from which the database will be restored. + (Added in 2.4) The format of the backup will be detected based on the target name. + Supported compression formats for dump and restore are: .bz2, .gz, and .xz + Supported formats for dump and restore are: .sql and .tar required: false default: present - choices: [ "present", "absent" ] + choices: [ "present", "absent", "dump", "restore" ] + target: + version_added: "2.4" + description: + - File to back up or restore from. Used when state is "dump" or "restore" + target_opts: + version_added: "2.4" + description: + - Further arguments for pg_dump or pg_restore. Used when state is "dump" or "restore" author: "Ansible Core Team" extends_documentation_fragment: - postgres @@ -85,12 +99,35 @@ EXAMPLES = ''' lc_collate: de_DE.UTF-8 lc_ctype: de_DE.UTF-8 template: template0 + +# Dump an existing database to a file +- postgresql_db: + name: acme + state: dump + target: /tmp/acme.sql + +# Dump an existing database to a file (with compression) +- postgresql_db: + name: acme + state: dump + target: /tmp/acme.sql.gz + +# Dump a single schema for an existing database +- postgresql_db: + name: acme + state: dump + target: /tmp/acme.sql + target_opts: "-n public" ''' HAS_PSYCOPG2 = False try: import psycopg2 import psycopg2.extras + import pipes + import subprocess + import os + except ImportError: pass else: @@ -205,6 +242,122 @@ def db_matches(cursor, db, owner, template, encoding, lc_collate, lc_ctype): else: return True +def db_dump(module, target, target_opts="", + db=None, + user=None, + password=None, + host=None, + port=None, + **kw): + + flags = login_flags(db, host, port, user, db_prefix=False) + cmd = module.get_bin_path('pg_dump', True) + comp_prog_path = None + + if os.path.splitext(target)[-1] == '.tar': + flags.append(' --format=t') + if os.path.splitext(target)[-1] == '.gz': + if module.get_bin_path('pigz'): + comp_prog_path = module.get_bin_path('pigz', True) + else: + comp_prog_path = module.get_bin_path('gzip', True) + elif os.path.splitext(target)[-1] == '.bz2': + comp_prog_path = module.get_bin_path('bzip2', True) + elif os.path.splitext(target)[-1] == '.xz': + comp_prog_path = module.get_bin_path('xz', True) + + cmd += "".join(flags) + if target_opts: + cmd += " {0} ".format(target_opts) + + if comp_prog_path: + cmd = '{0}|{1} > {2}'.format(cmd, comp_prog_path, pipes.quote(target)) + else: + cmd = '{0} > {1}'.format(cmd, pipes.quote(target)) + + return do_with_password(module, cmd, password) + +def db_restore(module, target, target_opts="", + db=None, + user=None, + password=None, + host=None, + port=None, + **kw): + + flags = login_flags(db, host, port, user) + comp_prog_path = None + cmd = module.get_bin_path('psql', True) + + if os.path.splitext(target)[-1] == '.sql': + flags.append(' --file={0}'.format(target)) + + elif os.path.splitext(target)[-1] == '.tar': + flags.append(' --format=Tar') + cmd = module.get_bin_path('pg_restore', True) + + elif os.path.splitext(target)[-1] == '.gz': + comp_prog_path = module.get_bin_path('zcat', True) + + elif os.path.splitext(target)[-1] == '.bz2': + comp_prog_path = module.get_bin_path('bzcat', True) + + elif os.path.splitext(target)[-1] == '.xz': + comp_prog_path = module.get_bin_path('xzcat', True) + + cmd += "".join(flags) + if target_opts: + cmd += " {0} ".format(target_opts) + + if comp_prog_path: + env = os.environ.copy() + if password: + env = {"PGPASSWORD": password} + p1 = subprocess.Popen([comp_prog_path, target], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p2 = subprocess.Popen(cmd, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=env) + (stdout2, stderr2) = p2.communicate() + p1.stdout.close() + p1.wait() + if p1.returncode != 0: + stderr1 = p1.stderr.read() + return p1.returncode, '', stderr1, 'cmd: ****' + else: + return p2.returncode, '', stderr2, 'cmd: ****' + else: + cmd = '{0} < {1}'.format(cmd, pipes.quote(target)) + + return do_with_password(module, cmd, password) + +def login_flags(db, host, port, user, db_prefix=True): + """ + returns a list of connection argument strings each prefixed + with a space and quoted where necessary to later be combined + in a single shell string with `"".join(rv)` + + db_prefix determines if "--dbname" is prefixed to the db argument, + since the argument was introduced in 9.3. + """ + flags = [] + if db: + if db_prefix: + flags.append(' --dbname={0}'.format(pipes.quote(db))) + else: + flags.append(' {0}'.format(pipes.quote(db))) + if host: + flags.append(' --host={0}'.format(host)) + if port: + flags.append(' --port={0}'.format(port)) + if user: + flags.append(' --username={0}'.format(user)) + return flags + +def do_with_password(module, cmd, password): + env = {} + if password: + env = {"PGPASSWORD": password} + rc, stderr, stdout = module.run_command(cmd, use_unsafe_shell=True, environ_update=env) + return rc, stderr, stdout, cmd + # =========================================== # Module execution. # @@ -218,7 +371,9 @@ def main(): encoding=dict(default=""), lc_collate=dict(default=""), lc_ctype=dict(default=""), - state=dict(default="present", choices=["absent", "present"]), + state=dict(default="present", choices=["absent", "present", "dump", "restore"]), + target=dict(default=""), + target_opts=dict(default=""), )) @@ -237,6 +392,8 @@ def main(): encoding = module.params["encoding"] lc_collate = module.params["lc_collate"] lc_ctype = module.params["lc_ctype"] + target = module.params["target"] + target_opts = module.params["target_opts"] state = module.params["state"] sslrootcert = module.params["ssl_rootcert"] changed = False @@ -257,12 +414,20 @@ def main(): # If a login_unix_socket is specified, incorporate it here. is_localhost = "host" not in kw or kw["host"] == "" or kw["host"] == "localhost" + if is_localhost and module.params["login_unix_socket"] != "": kw["host"] = module.params["login_unix_socket"] + if target == "": + target = "{0}/{1}.sql".format(os.getcwd(), db) + target = os.path.expanduser(target) + else: + target = os.path.expanduser(target) + try: pgutils.ensure_libs(sslrootcert=module.params.get('ssl_rootcert')) db_connection = psycopg2.connect(database="postgres", **kw) + # Enable autocommit so we can create databases if psycopg2.__version__ >= '2.4.2': db_connection.autocommit = True @@ -306,6 +471,19 @@ def main(): except SQLParseError: e = get_exception() module.fail_json(msg=str(e)) + + elif state in ("dump", "restore"): + method = state == "dump" and db_dump or db_restore + try: + rc, stdout, stderr, cmd = method(module, target, target_opts, db, **kw) + if rc != 0: + module.fail_json(msg=stderr, stdout=stdout, rc=rc, cmd=cmd) + else: + module.exit_json(changed=True, msg=stdout, stderr=stderr, rc=rc, cmd=cmd) + except SQLParseError: + e = get_exception() + module.fail_json(msg=str(e)) + except NotSupportedError: e = get_exception() module.fail_json(msg=str(e)) diff --git a/test/integration/targets/postgresql/tasks/main.yml b/test/integration/targets/postgresql/tasks/main.yml index 7c2bc860b7..fddc8a2039 100644 --- a/test/integration/targets/postgresql/tasks/main.yml +++ b/test/integration/targets/postgresql/tasks/main.yml @@ -754,6 +754,21 @@ that: - "result.stdout_lines[-1] == '(0 rows)'" +# dump/restore tests per format +# ============================================================ +- include: state_dump_restore.yml test_fixture=user file=dbdata.sql +- include: state_dump_restore.yml test_fixture=user file=dbdata.sql.gz +- include: state_dump_restore.yml test_fixture=user file=dbdata.sql.bz2 +- include: state_dump_restore.yml test_fixture=user file=dbdata.sql.xz +- include: state_dump_restore.yml test_fixture=user file=dbdata.tar +- include: state_dump_restore.yml test_fixture=user file=dbdata.tar.gz +- include: state_dump_restore.yml test_fixture=user file=dbdata.tar.bz2 +- include: state_dump_restore.yml test_fixture=user file=dbdata.tar.xz + +# dump/restore tests per other logins +# ============================================================ +- include: state_dump_restore.yml file=dbdata.tar test_fixture=admin + # # Cleanup # diff --git a/test/integration/targets/postgresql/tasks/state_dump_restore.yml b/test/integration/targets/postgresql/tasks/state_dump_restore.yml new file mode 100644 index 0000000000..deb2dc42bb --- /dev/null +++ b/test/integration/targets/postgresql/tasks/state_dump_restore.yml @@ -0,0 +1,113 @@ +# test code for state dump and restore for postgresql_db module +# copied from mysql_db/tasks/state_dump_import.yml +# (c) 2014, Wayne Rosario + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# ============================================================ +- set_fact: db_file_name="{{tmp_dir}}/{{file}}" + +- set_fact: + admin_str: "psql -U {{ pg_user }}" + admin_map: + name: "{{ db_name }}" + owner: "{{ db_user1 }}" + login_user: "{{ pg_user }}" + +- set_fact: + user_str: "env PGPASSWORD=password psql -h localhost -U {{ db_user1 }} {{ db_name }}" + user_map: + name: "{{ db_name }}" + target: "{{ db_file_name }}" + target_opts: "-n public" + owner: "{{ db_user1 }}" + login_host: "localhost" + login_user: "{{ db_user1 }}" + login_password: "password" + when: test_fixture == "user" + # "-n public" is required to work around pg_restore issues with plpgsql + +- set_fact: + user_str: "psql -U {{ pg_user }} {{ db_name }}" + user_map: + name: "{{ db_name }}" + target: "{{ db_file_name }}" + owner: "{{ db_user1 }}" + login_user: "{{ pg_user }}" + when: test_fixture == "admin" + +- set_fact: + sql_create: "create table employee(id int, name varchar(100));" + sql_insert: "insert into employee values (47,'Joe Smith');" + sql_select: "select * from employee;" + +- name: state dump/restore - create database + postgresql_db: "{{ admin_map | combine({'state': 'present'}) }}" + +- name: state dump/restore - create table employee + command: '{{ user_str }} -c "{{ sql_create }}"' + +- name: state dump/restore - insert data into table employee + command: '{{ user_str }} -c "{{ sql_insert }}"' + +- name: state dump/restore - file name should not exist + file: name={{ db_file_name }} state=absent + +- name: test state=dump to backup the database (expect changed=true) + postgresql_db: "{{ user_map | combine({'state': 'dump'}) }}" + register: result + become_user: "{{ pg_user }}" + become: True + +- name: assert output message backup the database + assert: + that: + - "result.changed == true" + +- name: assert database was backed up successfully + command: file {{ db_file_name }} + register: result + +- name: state dump/restore - remove database for restore + postgresql_db: "{{ user_map | combine({'state': 'absent'}) }}" + +- name: state dump/restore - re-create database + postgresql_db: "{{ admin_map | combine({'state': 'present'}) }}" + +- name: test state=restore to restore the database (expect changed=true) + postgresql_db: "{{ user_map | combine({'state': 'restore'}) }}" + register: result + become_user: "{{ pg_user }}" + become: True + +- name: assert output message restore the database + assert: { that: "result.changed == true" } + +- name: select data from table employee + command: '{{ user_str }} -c "{{ sql_select }}"' + register: result + +- name: assert data in database is from the restore database + assert: + that: + - "'47' in result.stdout" + - "'Joe Smith' in result.stdout" + +- name: state dump/restore - remove database name + postgresql_db: "{{ user_map | combine({'state': 'absent'}) }}" + +- name: remove file name + file: name={{ db_file_name }} state=absent