#!/usr/bin/env python # (c) 2012, Jan-Piet Mens # (c) 2012-2014, Michael DeHaan and others # (c) 2017 Ansible Project # # This file is part of Ansible # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Ansible is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . from __future__ import absolute_import, division, print_function __metaclass__ = type import datetime import glob import optparse import os import re import sys import warnings from collections import defaultdict from copy import deepcopy from distutils.version import LooseVersion from functools import partial from pprint import PrettyPrinter try: from html import escape as html_escape except ImportError: # Python-3.2 or later import cgi def html_escape(text, quote=True): return cgi.escape(text, quote) import jinja2 import yaml from jinja2 import Environment, FileSystemLoader from six import iteritems, string_types from ansible.errors import AnsibleError from ansible.module_utils._text import to_bytes, to_text from ansible.module_utils.common.collections import is_sequence from ansible.module_utils.parsing.convert_bool import boolean from ansible.plugins.loader import fragment_loader from ansible.utils import plugin_docs from ansible.utils.display import Display from ansible.utils._build_helpers import update_file_if_different ##################################################################################### # constants and paths # if a module is added in a version of Ansible older than this, don't print the version added information # in the module documentation because everyone is assumed to be running something newer than this already. TOO_OLD_TO_BE_NOTABLE = 1.3 # Get parent directory of the directory this script lives in MODULEDIR = os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, 'lib', 'ansible', 'modules' )) # The name of the DOCUMENTATION template EXAMPLE_YAML = os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml' )) _ITALIC = re.compile(r"I\(([^)]+)\)") _BOLD = re.compile(r"B\(([^)]+)\)") _MODULE = re.compile(r"M\(([^)]+)\)") _URL = re.compile(r"U\(([^)]+)\)") _LINK = re.compile(r"L\(([^)]+),([^)]+)\)") _CONST = re.compile(r"C\(([^)]+)\)") _RULER = re.compile(r"HORIZONTALLINE") DEPRECATED = b" (D)" pp = PrettyPrinter() display = Display() # kludge_ns gives us a kludgey way to set variables inside of loops that need to be visible outside # the loop. We can get rid of this when we no longer need to build docs with less than Jinja-2.10 # http://jinja.pocoo.org/docs/2.10/templates/#assignments # With Jinja-2.10 we can use jinja2's namespace feature, restoring the namespace template portion # of: fa5c0282a4816c4dd48e80b983ffc1e14506a1f5 NS_MAP = {} def to_kludge_ns(key, value): NS_MAP[key] = value return "" def from_kludge_ns(key): return NS_MAP[key] # The max filter was added in Jinja2-2.10. Until we can require that version, use this def do_max(seq): return max(seq) def rst_ify(text): ''' convert symbols like I(this is in italics) to valid restructured text ''' try: t = _ITALIC.sub(r"*\1*", text) t = _BOLD.sub(r"**\1**", t) t = _MODULE.sub(r":ref:`\1 <\1_module>`", t) t = _LINK.sub(r"`\1 <\2>`_", t) t = _URL.sub(r"\1", t) t = _CONST.sub(r"``\1``", t) t = _RULER.sub(r"------------", t) except Exception as e: raise AnsibleError("Could not process (%s) : %s" % (text, e)) return t def html_ify(text): ''' convert symbols like I(this is in italics) to valid HTML ''' if not isinstance(text, string_types): text = to_text(text) t = html_escape(text) t = _ITALIC.sub(r"\1", t) t = _BOLD.sub(r"\1", t) t = _MODULE.sub(r"\1", t) t = _URL.sub(r"\1", t) t = _LINK.sub(r"\1", t) t = _CONST.sub(r"\1", t) t = _RULER.sub(r"
", t) return t.strip() def rst_fmt(text, fmt): ''' helper for Jinja2 to do format strings ''' return fmt % (text) def rst_xline(width, char="="): ''' return a restructured text line of a given length ''' return char * width test_list = partial(is_sequence, include_strings=False) def normalize_options(value): """Normalize boolean option value.""" if value.get('type') == 'bool' and 'default' in value: try: value['default'] = boolean(value['default'], strict=True) except TypeError: pass return value def write_data(text, output_dir, outputname, module=None): ''' dumps module output to a file or the screen, as requested ''' if output_dir is not None: if module: outputname = outputname % module if not os.path.exists(output_dir): os.makedirs(output_dir) fname = os.path.join(output_dir, outputname) fname = fname.replace(".py", "") update_file_if_different(fname, to_bytes(text)) else: print(text) def get_plugin_info(module_dir, limit_to=None, verbose=False): ''' Returns information about plugins and the categories that they belong to :arg module_dir: file system path to the top of the plugin directory :kwarg limit_to: If given, this is a list of plugin names to generate information for. All other plugins will be ignored. :returns: Tuple of two dicts containing module_info, categories, and aliases and a set listing deprecated modules: :module_info: mapping of module names to information about them. The fields of the dict are: :path: filesystem path to the module :deprecated: boolean. True means the module is deprecated otherwise not. :aliases: set of aliases to this module name :metadata: The modules metadata (as recorded in the module) :doc: The documentation structure for the module :seealso: The list of dictionaries with references to related subjects :examples: The module's examples :returndocs: The module's returndocs :categories: maps category names to a dict. The dict contains at least one key, '_modules' which contains a list of module names in that category. Any other keys in the dict are subcategories with the same structure. ''' categories = dict() module_info = defaultdict(dict) # * windows powershell modules have documentation stubs in python docstring # format (they are not executed) so skip the ps1 format files # * One glob level for every module level that we're going to traverse files = ( glob.glob("%s/*.py" % module_dir) + glob.glob("%s/*/*.py" % module_dir) + glob.glob("%s/*/*/*.py" % module_dir) + glob.glob("%s/*/*/*/*.py" % module_dir) ) for module_path in files: # Do not list __init__.py files if module_path.endswith('__init__.py'): continue # Do not list blacklisted modules module = os.path.splitext(os.path.basename(module_path))[0] if module in plugin_docs.BLACKLIST['MODULE'] or module == 'base': continue # If requested, limit module documentation building only to passed-in # modules. if limit_to is not None and module.lower() not in limit_to: continue deprecated = False if module.startswith("_"): if os.path.islink(module_path): # Handle aliases source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0] module = module.replace("_", "", 1) aliases = module_info[source].get('aliases', set()) aliases.add(module) # In case we just created this via get()'s fallback module_info[source]['aliases'] = aliases continue else: # Handle deprecations module = module.replace("_", "", 1) deprecated = True # # Regular module to process # # use ansible core library to parse out doc metadata YAML and plaintext examples doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, fragment_loader, verbose=verbose) if metadata and 'removed' in metadata.get('status'): continue category = categories # Start at the second directory because we don't want the "vendor" mod_path_only = os.path.dirname(module_path[len(module_dir):]) primary_category = '' module_categories = [] # build up the categories that this module belongs to for new_cat in mod_path_only.split('/')[1:]: if new_cat not in category: category[new_cat] = dict() category[new_cat]['_modules'] = [] module_categories.append(new_cat) category = category[new_cat] category['_modules'].append(module) # the category we will use in links (so list_of_all_plugins can point to plugins/action_plugins/*' if module_categories: primary_category = module_categories[0] if 'options' in doc and doc['options'] is None: display.error("*** ERROR: DOCUMENTATION.options must be a dictionary/hash when used. ***") pos = getattr(doc, "ansible_pos", None) if pos is not None: display.error("Module position: %s, %d, %d" % doc.ansible_pos) doc['options'] = dict() for key, opt in doc.get('options', {}).items(): doc['options'][key] = normalize_options(opt) # save all the information module_info[module] = {'path': module_path, 'source': os.path.relpath(module_path, module_dir), 'deprecated': deprecated, 'aliases': module_info[module].get('aliases', set()), 'metadata': metadata, 'doc': doc, 'examples': examples, 'returndocs': returndocs, 'categories': module_categories, 'primary_category': primary_category, } # keep module tests out of becoming module docs if 'test' in categories: del categories['test'] return module_info, categories def generate_parser(): ''' generate an optparse parser ''' p = optparse.OptionParser( version='%prog 1.0', usage='usage: %prog [options] arg1 arg2', description='Generate module documentation from metadata', ) p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number") p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path") p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='module', help="The type of plugin (module, lookup, etc)") p.add_option("-T", "--template-dir", action="append", dest="template_dir", help="directory containing Jinja2 templates") p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type") p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files") p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules") p.add_option("-l", "--limit-to-modules", '--limit-to', action="store", dest="limit_to", default=None, help="Limit building module documentation to comma-separated list of plugins. Specify non-existing plugin name for no plugins.") p.add_option('-V', action='version', help='Show version number and exit') p.add_option('-v', '--verbose', dest='verbosity', default=0, action="count", help="verbose mode (increase number of 'v's for more)") return p def jinja2_environment(template_dir, typ, plugin_type): env = Environment(loader=FileSystemLoader(template_dir), variable_start_string="@{", variable_end_string="}@", trim_blocks=True) env.globals['xline'] = rst_xline # Can be removed (and template switched to use namespace) when we no longer need to build # with