From 8d086b171099f0a968f1fdd1b39706ec64a52f56 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Thu, 18 May 2017 15:38:22 -0500 Subject: Add storage sample environments Starts converting storage-related sample environments to the tool, and adds a few new ones for demonstration purposes. This has required the addition of a new category of parameter overrides in the tool. There are some parameters that are part of the public API of roles that should not normally be included in a sample environment for that role. Examples are EndpointMap and ServiceNetMap. Those are both passed into most (all?) roles, but their template defaults are not useful (both default to {}). Unless we are explicitly creating a sample environment that overrides those defaults we don't want them included. Parameters such as RoleName and RoleParameters are similar. We can't change them because they are part of the composable roles interface and that would break any existing custom roles, but we don't really want them included normally either. It's possible these could be made completely private, but there have been some very preliminary discussions about generating role samples that might actually want to set them. In order to avoid issues with editing the unit test file in editors that strip trailing whitespace, the minor formatting bug where params like EndpointMap had a trailing space after the name has also been fixed. Change-Id: If11f30c734bfbc17d463a9890c736d7477186fb9 --- environments/cinder-netapp-config.yaml | 4 + environments/puppet-ceph-external.yaml | 4 + environments/ssl/tls-endpoints-public-dns.yaml | 2 +- environments/ssl/tls-endpoints-public-ip.yaml | 2 +- environments/ssl/tls-everywhere-endpoints-dns.yaml | 2 +- environments/storage/cinder-netapp-config.yaml | 119 ++++++++++++++++++ environments/storage/cinder-nfs.yaml | 27 +++++ environments/storage/enable-ceph.yaml | 35 ++++++ environments/storage/external-ceph.yaml | 78 ++++++++++++ environments/storage/glance-nfs.yaml | 34 ++++++ sample-env-generator/storage.yaml | 133 +++++++++++++++++++++ tripleo_heat_templates/environment_generator.py | 40 +++++-- .../tests/test_environment_generator.py | 39 ++++++ 13 files changed, 503 insertions(+), 16 deletions(-) create mode 100644 environments/storage/cinder-netapp-config.yaml create mode 100644 environments/storage/cinder-nfs.yaml create mode 100644 environments/storage/enable-ceph.yaml create mode 100644 environments/storage/external-ceph.yaml create mode 100644 environments/storage/glance-nfs.yaml create mode 100644 sample-env-generator/storage.yaml diff --git a/environments/cinder-netapp-config.yaml b/environments/cinder-netapp-config.yaml index dfd15893..83ecbbe4 100644 --- a/environments/cinder-netapp-config.yaml +++ b/environments/cinder-netapp-config.yaml @@ -1,3 +1,7 @@ +# ************************************************************************************* +# DEPRECATED: Use tripleo-heat-templates/environments/storage/cinder-netapp-config.yaml +# instead. +# ************************************************************************************* # A Heat environment file which can be used to enable a # a Cinder NetApp backend, configured via puppet resource_registry: diff --git a/environments/puppet-ceph-external.yaml b/environments/puppet-ceph-external.yaml index 5f8b02ad..2f577c26 100644 --- a/environments/puppet-ceph-external.yaml +++ b/environments/puppet-ceph-external.yaml @@ -1,3 +1,7 @@ +# ****************************************************************************** +# DEPRECATED: Use tripleo-heat-templates/environments/storage/ceph-external.yaml +# instead. +# ****************************************************************************** # A Heat environment file which can be used to enable the # use of an externally managed Ceph cluster. resource_registry: diff --git a/environments/ssl/tls-endpoints-public-dns.yaml b/environments/ssl/tls-endpoints-public-dns.yaml index d1cab981..216afece 100644 --- a/environments/ssl/tls-endpoints-public-dns.yaml +++ b/environments/ssl/tls-endpoints-public-dns.yaml @@ -11,7 +11,7 @@ parameter_defaults: # Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. # Type: json - EndpointMap: + EndpointMap: AodhAdmin: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'} AodhInternal: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'} AodhPublic: {protocol: 'https', port: '13042', host: 'CLOUDNAME'} diff --git a/environments/ssl/tls-endpoints-public-ip.yaml b/environments/ssl/tls-endpoints-public-ip.yaml index cb41dc12..d216ab7f 100644 --- a/environments/ssl/tls-endpoints-public-ip.yaml +++ b/environments/ssl/tls-endpoints-public-ip.yaml @@ -11,7 +11,7 @@ parameter_defaults: # Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. # Type: json - EndpointMap: + EndpointMap: AodhAdmin: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'} AodhInternal: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'} AodhPublic: {protocol: 'https', port: '13042', host: 'IP_ADDRESS'} diff --git a/environments/ssl/tls-everywhere-endpoints-dns.yaml b/environments/ssl/tls-everywhere-endpoints-dns.yaml index 7ae7f3a0..63157ddd 100644 --- a/environments/ssl/tls-everywhere-endpoints-dns.yaml +++ b/environments/ssl/tls-everywhere-endpoints-dns.yaml @@ -11,7 +11,7 @@ parameter_defaults: # Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. # Type: json - EndpointMap: + EndpointMap: AodhAdmin: {protocol: 'https', port: '8042', host: 'CLOUDNAME'} AodhInternal: {protocol: 'https', port: '8042', host: 'CLOUDNAME'} AodhPublic: {protocol: 'https', port: '13042', host: 'CLOUDNAME'} diff --git a/environments/storage/cinder-netapp-config.yaml b/environments/storage/cinder-netapp-config.yaml new file mode 100644 index 00000000..4cdba09b --- /dev/null +++ b/environments/storage/cinder-netapp-config.yaml @@ -0,0 +1,119 @@ +# ******************************************************************* +# This file was created automatically by the sample environment +# generator. Developers should use `tox -e genconfig` to update it. +# Users are recommended to make changes to a copy of the file instead +# of the original, if any customizations are needed. +# ******************************************************************* +# title: Enable the Cinder NetApp Backend +# description: | +# A Heat environment file which can be used to enable a +# a Cinder NetApp backend, configured via puppet +parameter_defaults: + # + # Type: string + CinderNetappBackendName: tripleo_netapp + + # + # Type: string + CinderNetappControllerIps: '' + + # + # Type: string + CinderNetappCopyOffloadToolPath: '' + + # + # Type: string + CinderNetappEseriesHostType: linux_dm_mp + + # + # Type: string + CinderNetappHostType: '' + + # + # Mandatory. This parameter must be set by the user. + # Type: string + CinderNetappLogin: + + # + # Type: string + CinderNetappNfsMountOptions: '' + + # + # Type: string + CinderNetappNfsShares: '' + + # + # Type: string + CinderNetappNfsSharesConfig: /etc/cinder/shares.conf + + # + # Type: string + CinderNetappPartnerBackendName: '' + + # + # Mandatory. This parameter must be set by the user. + # Type: string + CinderNetappPassword: + + # + # Type: string + CinderNetappSaPassword: '' + + # + # Mandatory. This parameter must be set by the user. + # Type: string + CinderNetappServerHostname: + + # + # Type: string + CinderNetappServerPort: 80 + + # + # Type: string + CinderNetappSizeMultiplier: 1.2 + + # + # Type: string + CinderNetappStorageFamily: ontap_cluster + + # + # Type: string + CinderNetappStoragePools: '' + + # + # Type: string + CinderNetappStorageProtocol: nfs + + # + # Type: string + CinderNetappTransportType: http + + # + # Type: string + CinderNetappVfiler: '' + + # + # Type: string + CinderNetappVolumeList: '' + + # + # Type: string + CinderNetappVserver: '' + + # + # Type: string + CinderNetappWebservicePath: /devmgr/v2 + + # ****************************************************** + # Static parameters - these are values that must be + # included in the environment but should not be changed. + # ****************************************************** + # + # Type: boolean + CinderEnableNetappBackend: True + + # ********************* + # End static parameters + # ********************* +resource_registry: + OS::TripleO::ControllerExtraConfigPre: ../../puppet/extraconfig/pre_deploy/controller/cinder-netapp.yaml diff --git a/environments/storage/cinder-nfs.yaml b/environments/storage/cinder-nfs.yaml new file mode 100644 index 00000000..2de3e78c --- /dev/null +++ b/environments/storage/cinder-nfs.yaml @@ -0,0 +1,27 @@ +# ******************************************************************* +# This file was created automatically by the sample environment +# generator. Developers should use `tox -e genconfig` to update it. +# Users are recommended to make changes to a copy of the file instead +# of the original, if any customizations are needed. +# ******************************************************************* +# title: Enable Cinder NFS Backend +# description: | +# Configure and include this environment to enable the use of an NFS +# share as the backend for Cinder. +parameter_defaults: + # Whether to enable or not the Iscsi backend for Cinder + # Type: boolean + CinderEnableIscsiBackend: False + + # Whether to enable or not the NFS backend for Cinder + # Type: boolean + CinderEnableNfsBackend: True + + # Mount options for NFS mounts used by Cinder NFS backend. Effective when CinderEnableNfsBackend is true. + # Type: string + CinderNfsMountOptions: '' + + # NFS servers used by Cinder NFS backend. Effective when CinderEnableNfsBackend is true. + # Type: comma_delimited_list + CinderNfsServers: 192.168.122.1:/export/cinder + diff --git a/environments/storage/enable-ceph.yaml b/environments/storage/enable-ceph.yaml new file mode 100644 index 00000000..c629f74b --- /dev/null +++ b/environments/storage/enable-ceph.yaml @@ -0,0 +1,35 @@ +# ******************************************************************* +# This file was created automatically by the sample environment +# generator. Developers should use `tox -e genconfig` to update it. +# Users are recommended to make changes to a copy of the file instead +# of the original, if any customizations are needed. +# ******************************************************************* +# title: Enable Ceph Storage Backend +# description: | +# Include this environment to enable Ceph as the backend for +# Cinder, Nova, Gnocchi, and Glance. +parameter_defaults: + # The short name of the Cinder Backup backend to use. + # Type: string + CinderBackupBackend: rbd + + # Whether to enable or not the Iscsi backend for Cinder + # Type: boolean + CinderEnableIscsiBackend: False + + # Whether to enable or not the Rbd backend for Cinder + # Type: boolean + CinderEnableRbdBackend: True + + # The short name of the Glance backend to use. Should be one of swift, rbd, or file + # Type: string + GlanceBackend: rbd + + # The short name of the Gnocchi backend to use. Should be one of swift, rbd, or file + # Type: string + GnocchiBackend: rbd + + # Whether to enable or not the Rbd backend for Nova + # Type: boolean + NovaEnableRbdBackend: True + diff --git a/environments/storage/external-ceph.yaml b/environments/storage/external-ceph.yaml new file mode 100644 index 00000000..f1c9d516 --- /dev/null +++ b/environments/storage/external-ceph.yaml @@ -0,0 +1,78 @@ +# ******************************************************************* +# This file was created automatically by the sample environment +# generator. Developers should use `tox -e genconfig` to update it. +# Users are recommended to make changes to a copy of the file instead +# of the original, if any customizations are needed. +# ******************************************************************* +# title: Deploy Using an External Ceph Cluster +# description: | +# A Heat environment file which can be used to enable the +# use of an externally managed Ceph cluster. +parameter_defaults: + # The Ceph admin client key. Can be created with ceph-authtool --gen-print-key. + # Type: string + CephAdminKey: '' + + # The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring. + # Mandatory. This parameter must be set by the user. + # Type: string + CephClientKey: + + # + # Type: string + CephClientUserName: openstack + + # The Ceph cluster FSID. Must be a UUID. + # Mandatory. This parameter must be set by the user. + # Type: string + CephClusterFSID: + + # List of externally managed Ceph Mon Host IPs. Only used for external Ceph deployments. + # Type: string + CephExternalMonHost: '' + + # Whether to enable or not the Iscsi backend for Cinder + # Type: boolean + CinderEnableIscsiBackend: False + + # Whether to enable or not the Rbd backend for Cinder + # Type: boolean + CinderEnableRbdBackend: True + + # + # Type: string + CinderRbdPoolName: volumes + + # The short name of the Glance backend to use. Should be one of swift, rbd, or file + # Type: string + GlanceBackend: rbd + + # + # Type: string + GlanceRbdPoolName: images + + # The short name of the Gnocchi backend to use. Should be one of swift, rbd, or file + # Type: string + GnocchiBackend: rbd + + # + # Type: string + GnocchiRbdPoolName: metrics + + # Whether to enable or not the Rbd backend for Nova + # Type: boolean + NovaEnableRbdBackend: True + + # + # Type: string + NovaRbdPoolName: vms + + # The default features enabled when creating a block device image. Only applies to format 2 images. Set to '1' for Jewel clients using older Ceph servers. + # Type: string + RbdDefaultFeatures: '' + +resource_registry: + OS::TripleO::Services::CephClient: OS::Heat::None + OS::TripleO::Services::CephExternal: ../../puppet/services/ceph-external.yaml + OS::TripleO::Services::CephMon: OS::Heat::None + OS::TripleO::Services::CephOSD: OS::Heat::None diff --git a/environments/storage/glance-nfs.yaml b/environments/storage/glance-nfs.yaml new file mode 100644 index 00000000..3c139306 --- /dev/null +++ b/environments/storage/glance-nfs.yaml @@ -0,0 +1,34 @@ +# ******************************************************************* +# This file was created automatically by the sample environment +# generator. Developers should use `tox -e genconfig` to update it. +# Users are recommended to make changes to a copy of the file instead +# of the original, if any customizations are needed. +# ******************************************************************* +# title: Enable Glance NFS Backend +# description: | +# Configure and include this environment to enable the use of an NFS +# share as the backend for Glance. +parameter_defaults: + # NFS mount options for image storage (when GlanceNfsEnabled is true) + # Type: string + GlanceNfsOptions: intr,context=system_u:object_r:glance_var_lib_t:s0 + + # NFS share to mount for image storage (when GlanceNfsEnabled is true) + # Type: string + GlanceNfsShare: '' + + # ****************************************************** + # Static parameters - these are values that must be + # included in the environment but should not be changed. + # ****************************************************** + # The short name of the Glance backend to use. Should be one of swift, rbd, or file + # Type: string + GlanceBackend: file + + # When using GlanceBackend 'file', mount NFS share for image storage. + # Type: boolean + GlanceNfsEnabled: True + + # ********************* + # End static parameters + # ********************* diff --git a/sample-env-generator/storage.yaml b/sample-env-generator/storage.yaml new file mode 100644 index 00000000..aa0385cc --- /dev/null +++ b/sample-env-generator/storage.yaml @@ -0,0 +1,133 @@ +environments: + - + name: storage/enable-ceph + title: Enable Ceph Storage Backend + files: + puppet/services/cinder-volume.yaml: + parameters: + - CinderEnableIscsiBackend + - CinderEnableRbdBackend + puppet/services/cinder-backup.yaml: + parameters: + - CinderBackupBackend + puppet/services/nova-compute.yaml: + parameters: + - NovaEnableRbdBackend + puppet/services/glance-api.yaml: + parameters: + - GlanceBackend + puppet/services/gnocchi-api.yaml: + parameters: + - GnocchiBackend + sample_values: + CinderEnableIscsiBackend: False + CinderEnableRbdBackend: True + CinderBackupBackend: rbd + NovaEnableRbdBackend: True + GlanceBackend: rbd + GnocchiBackend: rbd + description: | + Include this environment to enable Ceph as the backend for + Cinder, Nova, Gnocchi, and Glance. + - + name: storage/cinder-nfs + title: Enable Cinder NFS Backend + files: + puppet/services/cinder-volume.yaml: + parameters: + - CinderNfsMountOptions + - CinderNfsServers + - CinderEnableNfsBackend + - CinderEnableIscsiBackend + sample_values: + CinderEnableNfsBackend: True + CinderEnableIscsiBackend: False + CinderNfsServers: '192.168.122.1:/export/cinder' + description: | + Configure and include this environment to enable the use of an NFS + share as the backend for Cinder. + - + name: storage/glance-nfs + title: Enable Glance NFS Backend + files: + puppet/services/glance-api.yaml: + parameters: + - GlanceBackend + - GlanceNfsEnabled + - GlanceNfsShare + - GlanceNfsOptions + sample_values: + GlanceBackend: file + GlanceNfsEnabled: True + static: + - GlanceBackend + - GlanceNfsEnabled + description: | + Configure and include this environment to enable the use of an NFS + share as the backend for Glance. + - + name: storage/external-ceph + title: Deploy Using an External Ceph Cluster + files: + puppet/services/nova-compute.yaml: + parameters: + - NovaRbdPoolName + - NovaEnableRbdBackend + - CephClientUserName + puppet/services/cinder-volume.yaml: + parameters: + - CinderRbdPoolName + - CinderEnableIscsiBackend + - CinderEnableRbdBackend + puppet/services/glance-api.yaml: + parameters: + - GlanceRbdPoolName + - GlanceBackend + puppet/services/gnocchi-api.yaml: + parameters: + - GnocchiBackend + puppet/services/gnocchi-base.yaml: + parameters: + - GnocchiRbdPoolName + puppet/services/ceph-external.yaml: + parameters: + - CephClusterFSID + - CephClientKey + - CephExternalMonHost + - RbdDefaultFeatures + puppet/services/ceph-base.yaml: + parameters: + - CephAdminKey + sample_values: + CinderEnableIscsiBackend: False + CinderEnableRbdBackend: True + NovaEnableRbdBackend: True + GlanceBackend: rbd + GnocchiBackend: rbd + NovaRbdPoolName: vms + CinderRbdPoolName: volumes + GlanceRbdPoolName: images + GnocchiRbdPoolName: metrics + CephClientUserName: openstack + CephAdminKey: '' + description: | + A Heat environment file which can be used to enable the + use of an externally managed Ceph cluster. + resource_registry: + OS::TripleO::Services::CephExternal: ../../puppet/services/ceph-external.yaml + OS::TripleO::Services::CephMon: OS::Heat::None + OS::TripleO::Services::CephClient: OS::Heat::None + OS::TripleO::Services::CephOSD: OS::Heat::None + - + name: storage/cinder-netapp-config + title: Enable the Cinder NetApp Backend + description: | + A Heat environment file which can be used to enable a + a Cinder NetApp backend, configured via puppet + files: + puppet/services/cinder-backend-netapp.yaml: + parameters: all + static: + - CinderEnableNetappBackend + resource_registry: + OS::TripleO::ControllerExtraConfigPre: ../../puppet/extraconfig/pre_deploy/controller/cinder-netapp.yaml diff --git a/tripleo_heat_templates/environment_generator.py b/tripleo_heat_templates/environment_generator.py index 659a7d57..b3e327f0 100755 --- a/tripleo_heat_templates/environment_generator.py +++ b/tripleo_heat_templates/environment_generator.py @@ -22,7 +22,7 @@ import yaml _PARAM_FORMAT = u""" # %(description)s %(mandatory)s# Type: %(type)s - %(name)s: %(default)s + %(name)s:%(default)s """ _STATIC_MESSAGE_START = ( ' # ******************************************************\n' @@ -44,7 +44,14 @@ _FILE_HEADER = ( ) # Certain parameter names can't be changed, but shouldn't be shown because # they are never intended for direct user input. -_PRIVATE_OVERRIDES = ['server', 'servers', 'NodeIndex'] +_PRIVATE_OVERRIDES = ['server', 'servers', 'NodeIndex', 'DefaultPasswords'] +# Hidden params are not included by default when the 'all' option is used, +# but can be explicitly included by referencing them in sample_defaults or +# static. This allows us to generate sample environments using them when +# necessary, but they won't be improperly included by accident. +_HIDDEN_PARAMS = ['EndpointMap', 'RoleName', 'RoleParameters', + 'ServiceNetMap', + ] def _create_output_dir(target_file): @@ -64,6 +71,8 @@ def _generate_environment(input_env, parent_env=None): env.update(input_env) parameter_defaults = {} param_names = [] + sample_values = env.get('sample_values', {}) + static_names = env.get('static', []) for template_file, template_data in env['files'].items(): with open(template_file) as f: f_data = yaml.safe_load(f) @@ -71,6 +80,10 @@ def _generate_environment(input_env, parent_env=None): parameter_defaults.update(f_params) if template_data['parameters'] == 'all': new_names = [k for k, v in f_params.items()] + for hidden in _HIDDEN_PARAMS: + if (hidden not in (static_names + sample_values.keys()) and + hidden in new_names): + new_names.remove(hidden) else: new_names = template_data['parameters'] missing_params = [name for name in new_names @@ -82,7 +95,6 @@ def _generate_environment(input_env, parent_env=None): env['name'])) param_names += new_names - static_names = env.get('static', []) static_defaults = {k: v for k, v in parameter_defaults.items() if k in param_names and k in static_names @@ -93,7 +105,8 @@ def _generate_environment(input_env, parent_env=None): not k.startswith('_') and k not in static_names } - for k, v in env.get('sample_values', {}).items(): + + for k, v in sample_values.items(): if k in parameter_defaults: parameter_defaults[k]['sample'] = v if k in static_defaults: @@ -108,17 +121,18 @@ def _generate_environment(input_env, parent_env=None): default = '' if value.get('sample') is not None: default = value['sample'] + # We ultimately cast this to str for output anyway + default = str(default) if default == '': default = "''" - try: - # If the default value is something like %index%, yaml won't - # parse the output correctly unless we wrap it in quotes. - # However, not all default values can be wrapped so we need to - # do it conditionally. - if default.startswith('%'): - default = "'%s'" % default - except AttributeError: - pass + # If the default value is something like %index%, yaml won't + # parse the output correctly unless we wrap it in quotes. + # However, not all default values can be wrapped so we need to + # do it conditionally. + if default.startswith('%'): + default = "'%s'" % default + if not default.startswith('\n'): + default = ' ' + default values = {'name': name, 'type': value['type'], diff --git a/tripleo_heat_templates/tests/test_environment_generator.py b/tripleo_heat_templates/tests/test_environment_generator.py index d0a622da..f4c4cdbf 100644 --- a/tripleo_heat_templates/tests/test_environment_generator.py +++ b/tripleo_heat_templates/tests/test_environment_generator.py @@ -34,6 +34,10 @@ parameters: default: 42 description: Bar description type: number + EndpointMap: + default: {} + description: Parameter that should not be included by default + type: json resources: # None ''' @@ -305,6 +309,41 @@ parameter_defaults: resource_registry: OS::TripleO::FakeResource: fake-filename.yaml +''', + }), + ('basic-hidden', + {'template': basic_template, + 'exception': None, + 'input_file': '''environments: + - + name: basic + title: Basic Environment + description: Basic description + files: + foo.yaml: + parameters: all + sample_values: + EndpointMap: |-2 + + foo: bar +''', + 'expected_output': '''# title: Basic Environment +# description: | +# Basic description +parameter_defaults: + # Bar description + # Type: number + BarParam: 42 + + # Parameter that should not be included by default + # Type: json + EndpointMap: + foo: bar + + # Foo description + # Type: string + FooParam: foo + ''', }), ('missing-param', -- cgit 1.2.3-korg