uri: Add form-urlencoded support to body_format (#37188)

* uri: Add form-urlencoded support to body_format

This PR adds form-urlencoded support so the user does not need to take
care of correctly encode input and have the same convenience as using
JSON.

This fixes #37182

* Various fixes

* Undo documentation improvements

No longer my problem

* Fix the remaining review comments

lib/ansible/modules/net_tools/basics/uri.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# (c) 2013, Romeo Theriault <romeot () hawaii.edu>
+# Copyright: (c) 2013, Romeo Theriault <romeot () hawaii.edu>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import absolute_import, division, print_function
@@ -40,14 +40,15 @@ options:
     description:
       - The body of the http request/response to the web service. If C(body_format) is set
         to 'json' it will take an already formatted JSON string or convert a data structure
-        into JSON.
+        into JSON. If C(body_format) is set to 'form-urlencoded' it will convert a dictionary
+        or list of tuples into an 'application/x-www-form-urlencoded' string. (Added in v2.7)
   body_format:
     description:
-      - The serialization format of the body. When set to json, encodes the
+      - The serialization format of the body. When set to C(json) or C(form-urlencoded), encodes the
         body argument, if needed, and automatically sets the Content-Type header accordingly.
         As of C(2.3) it is possible to override the `Content-Type` header, when
-        set to json via the I(headers) option.
+        set to C(json) or C(form-urlencoded) via the I(headers) option.
-    choices: [ "raw", "json" ]
+    choices: [ form-urlencoded, json, raw ]
     default: raw
     version_added: "2.0"
   method:
@@ -79,8 +80,8 @@ options:
         any redirects. Note that C(yes) and C(no) choices are accepted for backwards compatibility,
         where C(yes) is the equivalent of C(all) and C(no) is the equivalent of C(safe). C(yes) and C(no)
         are deprecated and will be removed in some future version of Ansible.
-    choices: [ all, none, safe ]
+    choices: [ all, 'none', safe ]
-    default: "safe"
+    default: safe
   creates:
     description:
       - A filename, when it already exists, this step will not be run.
@@ -89,8 +90,8 @@ options:
       - A filename, when it does not exist, this step will not be run.
   status_code:
     description:
-      - A valid, numeric, HTTP status code that signifies success of the
+      - A list of valid, numeric, HTTP status codes that signifies success of the
-        request. Can also be comma separated list of status codes.
+        request.
     default: 200
   timeout:
     description:
@@ -107,7 +108,7 @@ options:
     description:
         - Add custom HTTP headers to a request in the format of a YAML hash. As
           of C(2.3) supplying C(Content-Type) here will override the header
-          generated by supplying C(json) for I(body_format).
+          generated by supplying C(json) or C(form-urlencoded) for I(body_format).
     version_added: '2.1'
   others:
     description:
@@ -150,12 +151,8 @@ EXAMPLES = r'''
 - uri:
     url: http://www.example.com
     return_content: yes
-  register: webpage
+  register: this
-
+  failed_when: "'AWESOME' not in this.content"
-- name: Fail if AWESOME is not in the page content
-  fail:
-  when: "'AWESOME' not in webpage.content"
-
 
 - name: Create a JIRA issue
   uri:
@@ -174,10 +171,24 @@ EXAMPLES = r'''
 - uri:
     url: https://your.form.based.auth.example.com/index.php
     method: POST
-    body: "name=your_username&password=your_password&enter=Sign%20in"
+    body_format: form-urlencoded
+    body:
+      name: your_username
+      password: your_password
+      enter: Sign in
+    status_code: 302
+  register: login
+
+# Same, but now using a list of tuples
+- uri:
+    url: https://your.form.based.auth.example.com/index.php
+    method: POST
+    body_format: form-urlencoded
+    body:
+    - [ name, your_username ]
+    - [ password, your_password ]
+    - [ enter, Sign in ]
     status_code: 302
-    headers:
-      Content-Type: "application/x-www-form-urlencoded"
   register: login
 
 - uri:
@@ -185,17 +196,16 @@ EXAMPLES = r'''
     method: GET
     return_content: yes
     headers:
-      Cookie: "{{login.set_cookie}}"
+      Cookie: "{{ login.set_cookie }}"
 
 - name: Queue build of a project in Jenkins
   uri:
-    url: "http://{{ jenkins.host }}/job/{{ jenkins.job }}/build?token={{ jenkins.token }}"
+    url: http://{{ jenkins.host }}/job/{{ jenkins.job }}/build?token={{ jenkins.token }}
     method: GET
     user: "{{ jenkins.user }}"
     password: "{{ jenkins.password }}"
     force_basic_auth: yes
     status_code: 201
-
 '''
 
 RETURN = r'''
@@ -230,9 +240,10 @@ import shutil
 import tempfile
 import traceback
 
-
+from collections import Mapping, Sequence
 from ansible.module_utils.basic import AnsibleModule
-import ansible.module_utils.six as six
+from ansible.module_utils.six import iteritems, string_types
+from ansible.module_utils.six.moves.urllib.parse import urlencode, urlsplit
 from ansible.module_utils._text import to_native, to_text
 from ansible.module_utils.urls import fetch_url, url_argument_spec
 
@@ -290,7 +301,7 @@ def write_file(module, url, dest, content):
 
 
 def url_filename(url):
-    fn = os.path.basename(six.moves.urllib.parse.urlsplit(url)[2])
+    fn = os.path.basename(urlsplit(url)[2])
     if fn == '':
         return 'index.html'
     return fn
@@ -305,7 +316,7 @@ def absolute_location(url, location):
         return location
 
     elif location.startswith('/'):
-        parts = six.moves.urllib.parse.urlsplit(url)
+        parts = urlsplit(url)
         base = url.replace(parts[2], '')
         return '%s%s' % (base, location)
 
@@ -317,6 +328,39 @@ def absolute_location(url, location):
         return location
 
 
+def kv_list(data):
+    ''' Convert data into a list of key-value tuples '''
+    if data is None:
+        return None
+
+    if isinstance(data, Sequence):
+        return list(data)
+
+    if isinstance(data, Mapping):
+        return list(data.items())
+
+    raise TypeError('cannot form-urlencode body, expect list or dict')
+
+
+def form_urlencoded(body):
+    ''' Convert data into a form-urlencoded string '''
+    if isinstance(body, string_types):
+        return body
+
+    if isinstance(body, (Mapping, Sequence)):
+        result = []
+        # Turn a list of lists into a list of tupples that urlencode accepts
+        for key, values in kv_list(body):
+            if isinstance(values, string_types) or not isinstance(values, (Mapping, Sequence)):
+                values = [values]
+            for value in values:
+                if value is not None:
+                    result.append((to_text(key), to_text(value)))
+        return urlencode(result, doseq=True)
+
+    return body
+
+
 def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
     # is dest is set and is a directory, let's check if we get redirected and
     # set the filename from that url
@@ -373,9 +417,9 @@ def main():
         url_username=dict(type='str', aliases=['user']),
         url_password=dict(type='str', aliases=['password'], no_log=True),
         body=dict(type='raw'),
-        body_format=dict(type='str', default='raw', choices=['raw', 'json']),
+        body_format=dict(type='str', default='raw', choices=['form-urlencoded', 'json', 'raw']),
         method=dict(type='str', default='GET', choices=['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'PATCH', 'TRACE', 'CONNECT', 'REFRESH']),
-        return_content=dict(type='bool', default='no'),
+        return_content=dict(type='bool', default=False),
         follow_redirects=dict(type='str', default='safe', choices=['all', 'no', 'none', 'safe', 'urllib2', 'yes']),
         creates=dict(type='path'),
         removes=dict(type='path'),
@@ -406,16 +450,23 @@ def main():
 
     if body_format == 'json':
         # Encode the body unless its a string, then assume it is pre-formatted JSON
-        if not isinstance(body, six.string_types):
+        if not isinstance(body, string_types):
             body = json.dumps(body)
-        lower_header_keys = [key.lower() for key in dict_headers]
+        if 'content-type' not in [header.lower() for header in dict_headers]:
-        if 'content-type' not in lower_header_keys:
             dict_headers['Content-Type'] = 'application/json'
+    elif body_format == 'form-urlencoded':
+        if not isinstance(body, string_types):
+            try:
+                body = form_urlencoded(body)
+            except ValueError as e:
+                module.fail_json(msg='failed to parse body as form_urlencoded: %s' % to_native(e))
+        if 'content-type' not in [header.lower() for header in dict_headers]:
+            dict_headers['Content-Type'] = 'application/x-www-form-urlencoded'
 
     # TODO: Deprecated section.  Remove in Ansible 2.9
     # Grab all the http headers. Need this hack since passing multi-values is
     # currently a bit ugly. (e.g. headers='{"Content-Type":"application/json"}')
-    for key, value in six.iteritems(module.params):
+    for key, value in iteritems(module.params):
         if key.startswith("HEADER_"):
             module.deprecate('Supplying headers via HEADER_* is deprecated. Please use `headers` to'
                              ' supply headers for the request', version='2.9')
@@ -432,7 +483,7 @@ def main():
 
     if removes is not None:
         # do not run the command if the line contains removes=filename
-        # and the filename do not exists.  This allows idempotence
+        # and the filename does not exist.  This allows idempotence
         # of uri executions.
         if not os.path.exists(removes):
             module.exit_json(stdout="skipped, since '%s' does not exist" % removes, changed=False, rc=0)
@@ -463,7 +514,7 @@ def main():
     # In python3, the headers are title cased.  Lowercase them to be
     # compatible with the python2 behaviour.
     uresp = {}
-    for key, value in six.iteritems(resp):
+    for key, value in iteritems(resp):
         ukey = key.replace("-", "_").lower()
         uresp[ukey] = value
 

test/integration/targets/uri/tasks/main.yml

@@ -334,6 +334,62 @@
   register: result
   failed_when: result.json.headers['Content-Type'] != 'text/json'
 
+- name: Validate body_format form-urlencoded using dicts works
+  uri:
+    url: https://{{ httpbin_host }}/post
+    method: POST
+    body:
+      user: foo
+      password: bar!#@ |&82$M
+      submit: Sign in
+    body_format: form-urlencoded
+    return_content: yes
+  register: result
+
+- name: Assert form-urlencoded dict input
+  assert:
+    that:
+    - result is successful
+    - result.json.headers['Content-Type'] == 'application/x-www-form-urlencoded'
+    - result.json.form.password == 'bar!#@ |&82$M'
+
+- name: Validate body_format form-urlencoded using lists works
+  uri:
+    url: https://{{ httpbin_host }}/post
+    method: POST
+    body:
+      - [ user, foo ]
+      - [ password, bar!#@ |&82$M ]
+      - [ submit, Sign in ]
+    body_format: form-urlencoded
+    return_content: yes
+  register: result
+
+- name: Assert form-urlencoded list input
+  assert:
+    that:
+    - result is successful
+    - result.json.headers['Content-Type'] == 'application/x-www-form-urlencoded'
+    - result.json.form.password == 'bar!#@ |&82$M'
+
+- name: Validate body_format form-urlencoded of invalid input fails
+  uri:
+    url: https://{{ httpbin_host }}/post
+    method: POST
+    body:
+    - foo
+    - bar: baz
+    body_format: form-urlencoded
+    return_content: yes
+  register: result
+  ignore_errors: yes
+
+- name: Assert invalid input fails
+  assert:
+    that:
+    - result is failure
+    - "'failed to parse body as form_urlencoded: too many values to unpack' in result.msg"
+
 - name: Test client cert auth, no certs
   uri:
     url: "https://ansible.http.tests/ssl_client_verify"