diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | .testr.conf | 4 | ||||
| -rw-r--r-- | environments/predictable-placement/custom-hostnames.yaml | 33 | ||||
| -rw-r--r-- | sample-env-generator/README.rst | 149 | ||||
| -rw-r--r-- | sample-env-generator/sample-environments.yaml | 17 | ||||
| -rw-r--r-- | test-requirements.txt | 8 | ||||
| -rw-r--r-- | tox.ini | 10 | ||||
| -rw-r--r-- | tripleo_heat_templates/__init__.py | 0 | ||||
| -rwxr-xr-x | tripleo_heat_templates/environment_generator.py | 189 | ||||
| -rw-r--r-- | tripleo_heat_templates/tests/__init__.py | 0 | ||||
| -rw-r--r-- | tripleo_heat_templates/tests/test_environment_generator.py | 396 |
11 files changed, 808 insertions, 0 deletions
@@ -22,8 +22,10 @@ lib64 pip-log.txt # Unit test / coverage reports +cover .coverage .tox +.testrepository nosetests.xml # Translations diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..5837838f --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 OS_LOG_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./tripleo_heat_templates ./tripleo_heat_templates $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/environments/predictable-placement/custom-hostnames.yaml b/environments/predictable-placement/custom-hostnames.yaml new file mode 100644 index 00000000..0d9d520b --- /dev/null +++ b/environments/predictable-placement/custom-hostnames.yaml @@ -0,0 +1,33 @@ +# ******************************************************************* +# This file was created automatically by the sample environment +# generator. Developers should use `tox -e genconfig` to update it. +# Users are recommended to make changes to a copy of the file instead +# of the original, if any customizations are needed. +# ******************************************************************* +# title: Custom Hostnames +# description: | +# Hostname format for each role +# Note %index% is translated into the index of the node, e.g 0/1/2 etc +# and %stackname% is replaced with OS::stack_name in the template below. +# If you want to use the heat generated names, pass '' (empty string). +parameter_defaults: + # Format for BlockStorage node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud + # Type: string + BlockStorageHostnameFormat: '%stackname%-blockstorage-%index%' + + # Format for CephStorage node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud + # Type: string + CephStorageHostnameFormat: '%stackname%-cephstorage-%index%' + + # Format for Compute node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud + # Type: string + ComputeHostnameFormat: '%stackname%-novacompute-%index%' + + # Format for Controller node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud + # Type: string + ControllerHostnameFormat: '%stackname%-controller-%index%' + + # Format for ObjectStorage node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud + # Type: string + ObjectStorageHostnameFormat: '%stackname%-objectstorage-%index%' + diff --git a/sample-env-generator/README.rst b/sample-env-generator/README.rst new file mode 100644 index 00000000..71e9810a --- /dev/null +++ b/sample-env-generator/README.rst @@ -0,0 +1,149 @@ +Sample Environment Generator +---------------------------- + +This is a tool to automate the generation of our sample environment +files. It takes a yaml file as input, and based on the environments +defined in that file generates a number of sample environment files +from the parameters in the Heat templates. + +Usage +===== + +The simplest case is when an existing sample environment needs to be +updated to reflect changes in the templates. Use the tox ``genconfig`` +target to do this:: + + tox -e genconfig + +.. note:: The tool should be run from the root directory of the + ``tripleo-heat-templates`` project. + +If a new sample environment is needed, it should be added to the +``sample-env-generator/sample-environments.yaml`` file. The existing +entries in the file can be used as examples, and a more detailed +explanation of the different available keys is below: + +- **name**: the output file will be this name + .yaml, in the + ``environments`` directory. +- **title**: a human-readable title for the environment. +- **description**: A description of the environment. Will be included + as a comment at the top of the sample file. +- **files**: The Heat templates containing the parameter definitions + for the environment. Should be specified as a path relative to the + root of the ``tripleo-heat-templates`` project. For example: + ``puppet/extraconfig/tls/tls-cert-inject.yaml:``. Each filename + should be a YAML dictionary that contains a ``parameters`` entry. +- **parameters**: There should be one ``parameters`` entry per file in the + ``files`` section (see the example configuration below). + This can be either a list of parameters related to + the environment, which is necessary for templates like + overcloud.yaml, or the string 'all', which indicates that all + parameters from the file should be included. +- **static**: Can be used to specify that certain parameters must + not be changed. Examples would be the EnableSomething params + in the templates. When writing a sample config for Something, + ``EnableSomething: True`` would be a static param, since it + would be nonsense to include the environment with it set to any other + value. +- **sample_values**: Sometimes it is useful to include a sample value + for a parameter that is not the parameter's actual default. + An example of this is the SSLCertificate param in the enable-tls + environment file. +- **resource_registry**: Many environments also need to pass + resource_registry entries when they are used. This can be used + to specify that in the configuration file. + +Some behavioral notes: + +- Parameters without default values will be marked as mandatory to indicate + that the user must set a value for them. +- It is no longer recommended to set parameters using the ``parameters`` + section. Instead, all parameters should be set as ``parameter_defaults`` + which will work regardless of whether the parameter is top-level or nested. + Therefore, the tool will always set parameters in the ``parameter_defaults`` + section. +- Parameters whose name begins with the _ character are treated as private. + This indicates that the parameter value will be passed in from another + template and does not need to be exposed directly to the user. + +If adding a new environment, don't forget to add the new file to the +git repository so it will be included with the review. + +Example +======= + +Given a Heat template named ``example.yaml`` that looks like:: + + parameters: + EnableExample: + default: False + description: Enable the example feature + type: boolean + ParamOne: + default: one + description: First example param + type: string + ParamTwo: + description: Second example param + type: number + _PrivateParam: + default: does not matter + description: Will not show up + type: string + +And an environment generator entry that looks like:: + + environments: + - + name: example + title: Example Environment + description: | + An example environment demonstrating how to use the sample + environment generator. This text will be included at the top + of the generated file as a comment. + files: + example.yaml: + parameters: all + sample_values: + EnableExample: True + static: + - EnableExample + resource_registry: + OS::TripleO::ExampleData: ../extraconfig/example.yaml + +The generated environment file would look like:: + + # ******************************************************************* + # This file was created automatically by the sample environment + # generator. Developers should use `tox -e genconfig` to update it. + # Users are recommended to make changes to a copy of the file instead + # of the original, if any customizations are needed. + # ******************************************************************* + # title: Example Environment + # description: | + # An example environment demonstrating how to use the sample + # environment generator. This text will be included at the top + # of the generated file as a comment. + parameter_defaults: + # First example param + # Type: string + ParamOne: one + + # Second example param + # Mandatory. This parameter must be set by the user. + # Type: number + ParamTwo: <None> + + # ****************************************************** + # Static parameters - these are values that must be + # included in the environment but should not be changed. + # ****************************************************** + # Enable the example feature + # Type: boolean + EnableExample: True + + # ********************* + # End static parameters + # ********************* + resource_registry: + OS::TripleO::ExampleData: ../extraconfig/example.yaml diff --git a/sample-env-generator/sample-environments.yaml b/sample-env-generator/sample-environments.yaml new file mode 100644 index 00000000..ffda7aca --- /dev/null +++ b/sample-env-generator/sample-environments.yaml @@ -0,0 +1,17 @@ +environments: + - + name: predictable-placement/custom-hostnames + title: Custom Hostnames + files: + overcloud.yaml: + parameters: + - ControllerHostnameFormat + - ComputeHostnameFormat + - BlockStorageHostnameFormat + - ObjectStorageHostnameFormat + - CephStorageHostnameFormat + description: | + Hostname format for each role + Note %index% is translated into the index of the node, e.g 0/1/2 etc + and %stackname% is replaced with OS::stack_name in the template below. + If you want to use the heat generated names, pass '' (empty string). diff --git a/test-requirements.txt b/test-requirements.txt index 76f03d75..df5af85d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,3 +7,11 @@ six>=1.9.0 # MIT sphinx!=1.6.1,>=1.5.1 # BSD oslosphinx>=4.7.0 # Apache-2.0 reno!=2.3.1,>=1.8.0 # Apache-2.0 +coverage>=4.0,!=4.4 # Apache-2.0 +fixtures>=3.0.0 # Apache-2.0/BSD +python-subunit>=0.0.18 # Apache-2.0/BSD +testrepository>=0.0.18 # Apache-2.0/BSD +testscenarios>=0.4 # Apache-2.0/BSD +testtools>=1.4.0 # MIT +mock>=2.0 # BSD +oslotest>=1.10.0 # Apache-2.0 @@ -1,12 +1,14 @@ [tox] minversion = 1.6 skipsdist = True +envlist = py35,py27,pep8 [testenv] usedevelop = True install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:venv] commands = {posargs} @@ -22,3 +24,11 @@ commands = python ./tools/process-templates.py [testenv:releasenotes] commands = bash -c tools/releasenotes_tox.sh + +[testenv:cover] +commands = python setup.py test --coverage --coverage-package-name=tripleo_heat_templates --testr-args='{posargs}' + +[testenv:genconfig] +commands = + python ./tools/process-templates.py + python ./tripleo_heat_templates/environment_generator.py sample-env-generator/sample-environments.yaml diff --git a/tripleo_heat_templates/__init__.py b/tripleo_heat_templates/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tripleo_heat_templates/__init__.py diff --git a/tripleo_heat_templates/environment_generator.py b/tripleo_heat_templates/environment_generator.py new file mode 100755 index 00000000..e2f48720 --- /dev/null +++ b/tripleo_heat_templates/environment_generator.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python + +# Copyright 2015 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import errno +import os +import sys +import yaml + + +_PARAM_FORMAT = u""" # %(description)s + %(mandatory)s# Type: %(type)s + %(name)s: %(default)s +""" +_STATIC_MESSAGE_START = ( + ' # ******************************************************\n' + ' # Static parameters - these are values that must be\n' + ' # included in the environment but should not be changed.\n' + ' # ******************************************************\n' + ) +_STATIC_MESSAGE_END = (' # *********************\n' + ' # End static parameters\n' + ' # *********************\n' + ) +_FILE_HEADER = ( + '# *******************************************************************\n' + '# This file was created automatically by the sample environment\n' + '# generator. Developers should use `tox -e genconfig` to update it.\n' + '# Users are recommended to make changes to a copy of the file instead\n' + '# of the original, if any customizations are needed.\n' + '# *******************************************************************\n' + ) +# Certain parameter names can't be changed, but shouldn't be shown because +# they are never intended for direct user input. +_PRIVATE_OVERRIDES = ['server', 'servers', 'NodeIndex'] + + +def _create_output_dir(target_file): + try: + os.makedirs(os.path.dirname(target_file)) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + + +def _generate_environment(input_env, parent_env=None): + if parent_env is None: + parent_env = {} + env = dict(parent_env) + env.update(input_env) + parameter_defaults = {} + param_names = [] + for template_file, template_data in env['files'].items(): + with open(template_file) as f: + f_data = yaml.safe_load(f) + f_params = f_data['parameters'] + parameter_defaults.update(f_params) + if template_data['parameters'] == 'all': + new_names = [k for k, v in f_params.items()] + else: + new_names = template_data['parameters'] + missing_params = [name for name in new_names + if name not in f_params] + if missing_params: + raise RuntimeError('Did not find specified parameter names %s ' + 'in file %s for environment %s' % + (missing_params, template_file, + env['name'])) + param_names += new_names + + static_names = env.get('static', []) + static_defaults = {k: v for k, v in parameter_defaults.items() + if k in param_names and + k in static_names + } + parameter_defaults = {k: v for k, v in parameter_defaults.items() + if k in param_names and + k not in _PRIVATE_OVERRIDES and + not k.startswith('_') and + k not in static_names + } + for k, v in env.get('sample_values', {}).items(): + if k in parameter_defaults: + parameter_defaults[k]['sample'] = v + if k in static_defaults: + static_defaults[k]['sample'] = v + + def write_sample_entry(f, name, value): + default = value.get('default') + mandatory = '' + if default is None: + mandatory = ('# Mandatory. This parameter must be set by the ' + 'user.\n ') + default = '<None>' + if value.get('sample') is not None: + default = value['sample'] + if default == '': + default = "''" + try: + # If the default value is something like %index%, yaml won't + # parse the output correctly unless we wrap it in quotes. + # However, not all default values can be wrapped so we need to + # do it conditionally. + if default.startswith('%'): + default = "'%s'" % default + except AttributeError: + pass + + values = {'name': name, + 'type': value['type'], + 'description': + value.get('description', '').rstrip().replace('\n', + '\n # '), + 'default': default, + 'mandatory': mandatory, + } + f.write(_PARAM_FORMAT % values + '\n') + + target_file = os.path.join('environments', env['name'] + '.yaml') + _create_output_dir(target_file) + with open(target_file, 'w') as env_file: + env_file.write(_FILE_HEADER) + # TODO(bnemec): Once Heat allows the title and description to live in + # the environment itself, uncomment these entries and make them + # top-level keys in the YAML. + env_title = env.get('title', '') + env_file.write(u'# title: %s\n' % env_title) + env_desc = env.get('description', '') + env_file.write(u'# description: |\n') + for line in env_desc.splitlines(): + env_file.write(u'# %s\n' % line) + + if parameter_defaults: + env_file.write(u'parameter_defaults:\n') + for name, value in sorted(parameter_defaults.items()): + write_sample_entry(env_file, name, value) + if static_defaults: + env_file.write(_STATIC_MESSAGE_START) + for name, value in sorted(static_defaults.items()): + write_sample_entry(env_file, name, value) + if static_defaults: + env_file.write(_STATIC_MESSAGE_END) + + if env.get('resource_registry'): + env_file.write(u'resource_registry:\n') + for res, value in sorted(env.get('resource_registry', {}).items()): + env_file.write(u' %s: %s\n' % (res, value)) + print('Wrote sample environment "%s"' % target_file) + + for e in env.get('children', []): + _generate_environment(e, env) + + +def generate_environments(config_file): + with open(config_file) as f: + config = yaml.safe_load(f) + for env in config['environments']: + _generate_environment(env) + + +def usage(exit_code=1): + print('Usage: %s <filename.yaml>' % sys.argv[0]) + sys.exit(exit_code) + + +def main(): + try: + config_file = sys.argv[1] + except IndexError: + usage() + generate_environments(config_file) + + +if __name__ == '__main__': + main() diff --git a/tripleo_heat_templates/tests/__init__.py b/tripleo_heat_templates/tests/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tripleo_heat_templates/tests/__init__.py diff --git a/tripleo_heat_templates/tests/test_environment_generator.py b/tripleo_heat_templates/tests/test_environment_generator.py new file mode 100644 index 00000000..d0a622da --- /dev/null +++ b/tripleo_heat_templates/tests/test_environment_generator.py @@ -0,0 +1,396 @@ +# Copyright 2015 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import io +import tempfile + +import mock +from oslotest import base +import six +import testscenarios + +from tripleo_heat_templates import environment_generator + +load_tests = testscenarios.load_tests_apply_scenarios + +basic_template = ''' +parameters: + FooParam: + default: foo + description: Foo description + type: string + BarParam: + default: 42 + description: Bar description + type: number +resources: + # None +''' +basic_private_template = ''' +parameters: + FooParam: + default: foo + description: Foo description + type: string + _BarParam: + default: 42 + description: Bar description + type: number +resources: + # None +''' +mandatory_template = ''' +parameters: + FooParam: + description: Mandatory param + type: string +resources: + # None +''' +index_template = ''' +parameters: + FooParam: + description: Param with %index% as its default + type: string + default: '%index%' +resources: + # None +''' +multiline_template = ''' +parameters: + FooParam: + description: | + Parameter with + multi-line description + type: string + default: '' +resources: + # None +''' + + +class GeneratorTestCase(base.BaseTestCase): + content_scenarios = [ + ('basic', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Bar description + # Type: number + BarParam: 42 + + # Foo description + # Type: string + FooParam: foo + +''', + }), + ('basic-one-param', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: + - FooParam +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Foo description + # Type: string + FooParam: foo + +''', + }), + ('basic-static-param', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all + static: + - BarParam +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Foo description + # Type: string + FooParam: foo + + # ****************************************************** + # Static parameters - these are values that must be + # included in the environment but should not be changed. + # ****************************************************** + # Bar description + # Type: number + BarParam: 42 + + # ********************* + # End static parameters + # ********************* +''', + }), + ('basic-static-param-sample', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all + static: + - BarParam + sample_values: + BarParam: 1 + FooParam: '' +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Foo description + # Type: string + FooParam: '' + + # ****************************************************** + # Static parameters - these are values that must be + # included in the environment but should not be changed. + # ****************************************************** + # Bar description + # Type: number + BarParam: 1 + + # ********************* + # End static parameters + # ********************* +''', + }), + ('basic-private', + {'template': basic_private_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Foo description + # Type: string + FooParam: foo + +''', + }), + ('mandatory', + {'template': mandatory_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Mandatory param + # Mandatory. This parameter must be set by the user. + # Type: string + FooParam: <None> + +''', + }), + ('basic-sample', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all + sample_values: + FooParam: baz +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Bar description + # Type: number + BarParam: 42 + + # Foo description + # Type: string + FooParam: baz + +''', + }), + ('basic-resource-registry', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all + resource_registry: + OS::TripleO::FakeResource: fake-filename.yaml +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Bar description + # Type: number + BarParam: 42 + + # Foo description + # Type: string + FooParam: foo + +resource_registry: + OS::TripleO::FakeResource: fake-filename.yaml +''', + }), + ('missing-param', + {'template': basic_template, + 'exception': RuntimeError, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: + - SomethingNonexistent +''', + 'expected_output': None, + }), + ('percent-index', + {'template': index_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Param with %index% as its default + # Type: string + FooParam: '%index%' + +''', + }), + ('multi-line-desc', + {'template': multiline_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Parameter with + # multi-line description + # Type: string + FooParam: '' + +''', + }), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios( + cls.content_scenarios) + + def test_generator(self): + fake_input = io.StringIO(six.text_type(self.input_file)) + fake_template = io.StringIO(six.text_type(self.template)) + _, fake_output_path = tempfile.mkstemp() + fake_output = open(fake_output_path, 'w') + with mock.patch('tripleo_heat_templates.environment_generator.open', + create=True) as mock_open: + mock_open.side_effect = [fake_input, fake_template, fake_output] + if not self.exception: + environment_generator.generate_environments('ignored.yaml') + else: + self.assertRaises(self.exception, + environment_generator.generate_environments, + 'ignored.yaml') + return + expected = environment_generator._FILE_HEADER + self.expected_output + with open(fake_output_path) as f: + self.assertEqual(expected, f.read()) + +GeneratorTestCase.generate_scenarios() |