From 80c00d32388457f11af5e497d7c32e5a2ab6596b Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Tue, 29 Aug 2017 17:58:44 -0400 Subject: [PATCH] Create urlsplit filter (#28537) * Improve tests for uri filter * Create URL Split docs * Add urlsplit filter * Py3 compatibility * Use helper method and eliminate query options * Add options, cleanup output, fix tests * Update docs * Add parenthesis to boilerplate import * Add debug task to tests * Use exclude option to filter returned values * Filter out additional option for Python 3 --- docs/docsite/rst/playbooks_filters.rst | 70 +++++++-- lib/ansible/plugins/filter/urlsplit.py | 42 ++++++ .../targets/filters/tasks/main.yml | 134 +++++++++++------- 3 files changed, 184 insertions(+), 62 deletions(-) create mode 100644 lib/ansible/plugins/filter/urlsplit.py diff --git a/docs/docsite/rst/playbooks_filters.rst b/docs/docsite/rst/playbooks_filters.rst index 534a9afacf..f27af3122f 100644 --- a/docs/docsite/rst/playbooks_filters.rst +++ b/docs/docsite/rst/playbooks_filters.rst @@ -332,7 +332,7 @@ output, use the ``parse_cli`` filter:: The ``parse_cli`` filter will load the spec file and pass the command output through, it returning JSON output. The spec file is a YAML yaml that defines -how to parse the CLI output. +how to parse the CLI output. The spec file should be valid formatted YAML. It defines how to parse the CLI output and return JSON data. Below is an example of a valid spec file that @@ -357,8 +357,8 @@ will parse the output from the ``show vlan`` command.:: The spec file above will return a JSON data structure that is a list of hashes with the parsed VLAN information. -The same command could be parsed into a hash by using the key and values -directives. Here is an example of how to parse the output into a hash +The same command could be parsed into a hash by using the key and values +directives. Here is an example of how to parse the output into a hash value using the same ``show vlan`` command.:: --- @@ -379,7 +379,7 @@ value using the same ``show vlan`` command.:: state_static: value: present -Another common use case for parsing CLI commands is to break a large command +Another common use case for parsing CLI commands is to break a large command into blocks that can parsed. This can be done using the ``start_block`` and ``end_block`` directives to break the command into blocks that can be parsed.:: @@ -594,6 +594,56 @@ which will produce this output: .. _other_useful_filters: +URL Split Filter +````````````````` + +.. versionadded:: 2.4 + +The ``urlsplit`` filter extracts the fragment, hostname, netloc, password, path, port, query, scheme, and username from an URL. With no arguments, returns a dictionary of all the fields:: + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('hostname') }} + # => 'www.acme.com' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('netloc') }} + # => 'user:password@www.acme.com:9000' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('username') }} + # => 'user' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('password') }} + # => 'password' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('path') }} + # => '/dir/index.html' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('port') }} + # => '9000' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('scheme') }} + # => 'http' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('query') }} + # => 'query=term' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('fragment') }} + # => 'fragment' + + {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit }} + # => + # { + # "fragment": "fragment", + # "hostname": "www.acme.com", + # "netloc": "user:password@www.acme.com:9000", + # "password": "password", + # "path": "/dir/index.html", + # "port": 9000, + # "query": "query=term", + # "scheme": "http", + # "username": "user" + # } + + + Other Useful Filters ```````````````````` @@ -681,7 +731,7 @@ To replace text in a string with regex, use the "regex_replace" filter:: # convert "localhost:80" to "localhost, 80" using named groups {{ 'localhost:80' | regex_replace('^(?P.+):(?P\\d+)$', '\\g, \\g') }} - + # convert "localhost:80" to "localhost" {{ 'localhost:80' | regex_replace(':80') }} @@ -717,26 +767,26 @@ To get permutations of a list:: - name: give me largest permutations (order matters) debug: msg="{{ [1,2,3,4,5]|permutations|list }}" - - name: give me permutations of sets of 3 + - name: give me permutations of sets of three debug: msg="{{ [1,2,3,4,5]|permutations(3)|list }}" Combinations always require a set size:: - - name: give me combinations for sets of 2 + - name: give me combinations for sets of two debug: msg="{{ [1,2,3,4,5]|combinations(2)|list }}" To get a list combining the elements of other lists use ``zip``:: - - name: give me list combo of 2 lists + - name: give me list combo of two lists debug: msg="{{ [1,2,3,4,5]|zip(['a','b','c','d','e','f'])|list }}" - - name: give me shortest combo of 2 lists + - name: give me shortest combo of two lists debug: msg="{{ [1,2,3]|zip(['a','b','c','d','e','f'])|list }}" To always exhaust all list use ``zip_longest``:: - - name: give me longest combo of 3 lists , fill with X + - name: give me longest combo of three lists , fill with X debug: msg="{{ [1,2,3]|zip_longest(['a','b','c','d','e','f'], [21, 22, 23], fillvalue='X')|list }}" diff --git a/lib/ansible/plugins/filter/urlsplit.py b/lib/ansible/plugins/filter/urlsplit.py new file mode 100644 index 0000000000..7076de0fbe --- /dev/null +++ b/lib/ansible/plugins/filter/urlsplit.py @@ -0,0 +1,42 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + + +from ansible.errors import AnsibleFilterError +from ansible.module_utils.six.moves.urllib.parse import urlsplit +from ansible.utils import helpers + + +def split_url(value, query='', alias='urlsplit'): + + results = helpers.object_to_dict(urlsplit(value), exclude=['count', 'index', 'geturl', 'encode']) + + # If a query is supplied, make sure it's valid then return the results. + # If no option is supplied, return the entire dictionary. + if query: + if query not in results: + raise AnsibleFilterError(alias + ': unknown URL component: %s' % query) + return results[query] + else: + return results + + +# ---- Ansible filters ---- +class FilterModule(object): + ''' URI filter ''' + + def filters(self): + return { + 'urlsplit': split_url + } diff --git a/test/integration/targets/filters/tasks/main.yml b/test/integration/targets/filters/tasks/main.yml index 0d5b7ec2ae..48f74c15ec 100644 --- a/test/integration/targets/filters/tasks/main.yml +++ b/test/integration/targets/filters/tasks/main.yml @@ -20,23 +20,29 @@ shell: echo hi register: some_registered_var -- debug: var=some_registered_var +- debug: + var: some_registered_var - name: Verify that we workaround a py26 json bug - template: src=py26json.j2 dest={{output_dir}}/py26json.templated mode=0644 + template: + src: py26json.j2 + dest: "{{ output_dir }}/py26json.templated" + mode: 0644 - name: 9851 - Verify that we don't trigger https://github.com/ansible/ansible/issues/9851 copy: - content: " [{{item|to_nice_json}}]" - dest: "{{output_dir}}/9851.out" + content: " [{{ item | to_nice_json }}]" + dest: "{{ output_dir }}/9851.out" with_items: - {"k": "Quotes \"'\n"} - name: 9851 - copy known good output into place - copy: src=9851.txt dest={{output_dir}}/9851.txt + copy: + src: 9851.txt + dest: "{{ output_dir }}/9851.txt" - name: 9851 - Compare generated json to known good - shell: diff -w {{output_dir}}/9851.out {{output_dir}}/9851.txt + shell: diff -w {{ output_dir }}/9851.out {{ output_dir }}/9851.txt register: diff_result_9851 - name: 9851 - verify generated file matches known good @@ -45,72 +51,78 @@ - 'diff_result_9851.stdout == ""' - name: fill in a basic template - template: src=foo.j2 dest={{output_dir}}/foo.templated mode=0644 + template: + src: foo.j2 + dest: "{{ output_dir }}/foo.templated" + mode: 0644 register: template_result - name: copy known good into place - copy: src=foo.txt dest={{output_dir}}/foo.txt + copy: + src: foo.txt + dest: "{{ output_dir }}/foo.txt" - name: compare templated file to known good - shell: diff -w {{output_dir}}/foo.templated {{output_dir}}/foo.txt + shell: diff -w {{ output_dir }}/foo.templated {{ output_dir }}/foo.txt register: diff_result - name: verify templated file matches known good assert: that: - - 'diff_result.stdout == ""' + - 'diff_result.stdout == ""' - name: Verify human_readable tags: "human_readable" assert: that: - - '"1.00 Bytes" == 1|human_readable' - - '"1.00 bits" == 1|human_readable(isbits=True)' - - '"10.00 KB" == 10240|human_readable' - - '"97.66 MB" == 102400000|human_readable' - - '"0.10 GB" == 102400000|human_readable(unit="G")' - - '"0.10 Gb" == 102400000|human_readable(isbits=True, unit="G")' + - '"1.00 Bytes" == 1|human_readable' + - '"1.00 bits" == 1|human_readable(isbits=True)' + - '"10.00 KB" == 10240|human_readable' + - '"97.66 MB" == 102400000|human_readable' + - '"0.10 GB" == 102400000|human_readable(unit="G")' + - '"0.10 Gb" == 102400000|human_readable(isbits=True, unit="G")' - name: Verify human_to_bytes tags: "human_to_bytes" assert: that: - - "{{'0'|human_to_bytes}} == 0" - - "{{'0.1'|human_to_bytes}} == 0" - - "{{'0.9'|human_to_bytes}} == 1" - - "{{'1'|human_to_bytes}} == 1" - - "{{'10.00 KB'|human_to_bytes}} == 10240" - - "{{ '11 MB'|human_to_bytes}} == 11534336" - - "{{ '1.1 GB'|human_to_bytes}} == 1181116006" - - "{{'10.00 Kb'|human_to_bytes(isbits=True)}} == 10240" + - "{{'0'|human_to_bytes}} == 0" + - "{{'0.1'|human_to_bytes}} == 0" + - "{{'0.9'|human_to_bytes}} == 1" + - "{{'1'|human_to_bytes}} == 1" + - "{{'10.00 KB'|human_to_bytes}} == 10240" + - "{{ '11 MB'|human_to_bytes}} == 11534336" + - "{{ '1.1 GB'|human_to_bytes}} == 1181116006" + - "{{'10.00 Kb'|human_to_bytes(isbits=True)}} == 10240" - name: Verify human_to_bytes (bad string) - tags: "human_to_bytes" - set_fact: bad_string="{{'10.00 foo'|human_to_bytes}}" + set_fact: + bad_string: "{{ '10.00 foo' | human_to_bytes }}" ignore_errors: yes - register: _ + tags: human_to_bytes + register: _human_bytes_test - name: Verify human_to_bytes (bad string) - tags: "human_to_bytes" + tags: human_to_bytes assert: - that: "{{_.failed}}" + that: "{{_human_bytes_test.failed}}" - name: Test extract assert: that: - - '"c" == 2 | extract(["a", "b", "c"])' - - '"b" == 1 | extract(["a", "b", "c"])' - - '"a" == 0 | extract(["a", "b", "c"])' + - '"c" == 2 | extract(["a", "b", "c"])' + - '"b" == 1 | extract(["a", "b", "c"])' + - '"a" == 0 | extract(["a", "b", "c"])' - name: Container lookups with extract assert: that: - - "'x' == [0]|map('extract',['x','y'])|list|first" - - "'y' == [1]|map('extract',['x','y'])|list|first" - - "42 == ['x']|map('extract',{'x':42,'y':31})|list|first" - - "31 == ['x','y']|map('extract',{'x':42,'y':31})|list|last" - - "'local' == ['localhost']|map('extract',hostvars,'ansible_connection')|list|first" - - "'local' == ['localhost']|map('extract',hostvars,['ansible_connection'])|list|first" + - "'x' == [0]|map('extract',['x','y'])|list|first" + - "'y' == [1]|map('extract',['x','y'])|list|first" + - "42 == ['x']|map('extract',{'x':42,'y':31})|list|first" + - "31 == ['x','y']|map('extract',{'x':42,'y':31})|list|last" + - "'local' == ['localhost']|map('extract',hostvars,'ansible_connection')|list|first" + - "'local' == ['localhost']|map('extract',hostvars,['ansible_connection'])|list|first" # map was added to jinja2 in version 2.7 when: "{{ ( lookup('pipe', '{{ ansible_python[\"executable\"] }} -c \"import jinja2; print(jinja2.__version__)\"') | version_compare('2.7', '>=') ) }}" @@ -120,22 +132,40 @@ that: - "users | json_query('[*].hosts[].host') == ['host_a', 'host_b', 'host_c', 'host_d']" -- name: "20379 - set_fact app_var_git_branch " - set_fact: - app_var_git_branch: multi-deployment-400-743 - -- name: "20379 - trigger a error in jmespath via json_query filter to test error handling" - debug: - msg: "{{ example_20379 | json_query('ApplicationVersions[].VersionLabel[] | [?starts_with(@, `multi`)]') }}" - ignore_errors: true - -- name: "20379 - Test errors related to https://github.com/ansible/ansible/issues/20379" - assert: - that: "example_20379 | json_query('ApplicationVersions[].VersionLabel[] | [?starts_with(@, '+app_var_git_branch+')] | [2:]') == multisdfsdf" - ignore_errors: true - - name: Test hash filter assert: that: - '"{{ "hash" | hash("sha1") }}" == "2346ad27d7568ba9896f1b7da6b5991251debdf2"' - '"{{ "café" | hash("sha1") }}" == "f424452a9673918c6f09b0cdd35b20be8e6ae7d7"' + +- debug: + var: "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit" + verbosity: 1 + tags: debug + +- name: Test urlsplit filter + assert: + that: + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('fragment') == 'fragment'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('hostname') == 'www.acme.com'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('netloc') == 'mary:MySecret@www.acme.com:9000'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('path') == '/dir/index.html'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('port') == 9000" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('query') == 'query=term'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('scheme') == 'http'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('username') == 'mary'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('password') == 'MySecret'" + - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit == { 'fragment': 'fragment', 'hostname': 'www.acme.com', 'netloc': 'mary:MySecret@www.acme.com:9000', 'password': 'MySecret', 'path': '/dir/index.html', 'port': 9000, 'query': 'query=term', 'scheme': 'http', 'username': 'mary' }" + + +- name: Test urlsplit filter bad argument + debug: + var: "'http://www.acme.com:9000/dir/index.html' | urlsplit('bad_filter')" + register: _bad_urlsplit_filter + ignore_errors: yes + +- name: Verify urlsplit filter showed an error message + assert: + that: + - _bad_urlsplit_filter | failed + - "'unknown URL component' in _bad_urlsplit_filter.msg"