Merge "Add storage sample environments"

13 files changed, 503 insertions, 16 deletions

diff --git a/environments/cinder-netapp-config.yaml b/environments/cinder-netapp-config.yaml
index dfd15893..83ecbbe4 100644
--- a/environments/cinder-netapp-config.yaml
+++ b/environments/cinder-netapp-config.yaml
@@ -1,3 +1,7 @@
+# *************************************************************************************
+# DEPRECATED: Use tripleo-heat-templates/environments/storage/cinder-netapp-config.yaml
+# instead.
+# *************************************************************************************
 # A Heat environment file which can be used to enable a
 # a Cinder NetApp backend, configured via puppet
 resource_registry:
diff --git a/environments/puppet-ceph-external.yaml b/environments/puppet-ceph-external.yaml
index 5f8b02ad..2f577c26 100644
--- a/environments/puppet-ceph-external.yaml
+++ b/environments/puppet-ceph-external.yaml
@@ -1,3 +1,7 @@
+# ******************************************************************************
+# DEPRECATED: Use tripleo-heat-templates/environments/storage/ceph-external.yaml
+# instead.
+# ******************************************************************************
 # A Heat environment file which can be used to enable the
 # use of an externally managed Ceph cluster.
 resource_registry:
diff --git a/environments/ssl/tls-endpoints-public-dns.yaml b/environments/ssl/tls-endpoints-public-dns.yaml
index d1cab981..216afece 100644
--- a/environments/ssl/tls-endpoints-public-dns.yaml
+++ b/environments/ssl/tls-endpoints-public-dns.yaml
@@ -11,7 +11,7 @@
 parameter_defaults:
   # Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry.
   # Type: json
-  EndpointMap: 
+  EndpointMap:
     AodhAdmin: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'}
     AodhInternal: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'}
     AodhPublic: {protocol: 'https', port: '13042', host: 'CLOUDNAME'}
diff --git a/environments/ssl/tls-endpoints-public-ip.yaml b/environments/ssl/tls-endpoints-public-ip.yaml
index cb41dc12..d216ab7f 100644
--- a/environments/ssl/tls-endpoints-public-ip.yaml
+++ b/environments/ssl/tls-endpoints-public-ip.yaml
@@ -11,7 +11,7 @@
 parameter_defaults:
   # Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry.
   # Type: json
-  EndpointMap: 
+  EndpointMap:
     AodhAdmin: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'}
     AodhInternal: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'}
     AodhPublic: {protocol: 'https', port: '13042', host: 'IP_ADDRESS'}
diff --git a/environments/ssl/tls-everywhere-endpoints-dns.yaml b/environments/ssl/tls-everywhere-endpoints-dns.yaml
index 7ae7f3a0..63157ddd 100644
--- a/environments/ssl/tls-everywhere-endpoints-dns.yaml
+++ b/environments/ssl/tls-everywhere-endpoints-dns.yaml
@@ -11,7 +11,7 @@
 parameter_defaults:
   # Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry.
   # Type: json
-  EndpointMap: 
+  EndpointMap:
     AodhAdmin: {protocol: 'https', port: '8042', host: 'CLOUDNAME'}
     AodhInternal: {protocol: 'https', port: '8042', host: 'CLOUDNAME'}
     AodhPublic: {protocol: 'https', port: '13042', host: 'CLOUDNAME'}
diff --git a/environments/storage/cinder-netapp-config.yaml b/environments/storage/cinder-netapp-config.yaml
new file mode 100644
index 00000000..4cdba09b
--- /dev/null
+++ b/environments/storage/cinder-netapp-config.yaml
@@ -0,0 +1,119 @@
+# *******************************************************************
+# This file was created automatically by the sample environment
+# generator. Developers should use `tox -e genconfig` to update it.
+# Users are recommended to make changes to a copy of the file instead
+# of the original, if any customizations are needed.
+# *******************************************************************
+# title: Enable the Cinder NetApp Backend
+# description: |
+#   A Heat environment file which can be used to enable a
+#   a Cinder NetApp backend, configured via puppet
+parameter_defaults:
+  # 
+  # Type: string
+  CinderNetappBackendName: tripleo_netapp
+
+  # 
+  # Type: string
+  CinderNetappControllerIps: ''
+
+  # 
+  # Type: string
+  CinderNetappCopyOffloadToolPath: ''
+
+  # 
+  # Type: string
+  CinderNetappEseriesHostType: linux_dm_mp
+
+  # 
+  # Type: string
+  CinderNetappHostType: ''
+
+  # 
+  # Mandatory. This parameter must be set by the user.
+  # Type: string
+  CinderNetappLogin: <None>
+
+  # 
+  # Type: string
+  CinderNetappNfsMountOptions: ''
+
+  # 
+  # Type: string
+  CinderNetappNfsShares: ''
+
+  # 
+  # Type: string
+  CinderNetappNfsSharesConfig: /etc/cinder/shares.conf
+
+  # 
+  # Type: string
+  CinderNetappPartnerBackendName: ''
+
+  # 
+  # Mandatory. This parameter must be set by the user.
+  # Type: string
+  CinderNetappPassword: <None>
+
+  # 
+  # Type: string
+  CinderNetappSaPassword: ''
+
+  # 
+  # Mandatory. This parameter must be set by the user.
+  # Type: string
+  CinderNetappServerHostname: <None>
+
+  # 
+  # Type: string
+  CinderNetappServerPort: 80
+
+  # 
+  # Type: string
+  CinderNetappSizeMultiplier: 1.2
+
+  # 
+  # Type: string
+  CinderNetappStorageFamily: ontap_cluster
+
+  # 
+  # Type: string
+  CinderNetappStoragePools: ''
+
+  # 
+  # Type: string
+  CinderNetappStorageProtocol: nfs
+
+  # 
+  # Type: string
+  CinderNetappTransportType: http
+
+  # 
+  # Type: string
+  CinderNetappVfiler: ''
+
+  # 
+  # Type: string
+  CinderNetappVolumeList: ''
+
+  # 
+  # Type: string
+  CinderNetappVserver: ''
+
+  # 
+  # Type: string
+  CinderNetappWebservicePath: /devmgr/v2
+
+  # ******************************************************
+  # Static parameters - these are values that must be
+  # included in the environment but should not be changed.
+  # ******************************************************
+  # 
+  # Type: boolean
+  CinderEnableNetappBackend: True
+
+  # *********************
+  # End static parameters
+  # *********************
+resource_registry:
+  OS::TripleO::ControllerExtraConfigPre: ../../puppet/extraconfig/pre_deploy/controller/cinder-netapp.yaml
diff --git a/environments/storage/cinder-nfs.yaml b/environments/storage/cinder-nfs.yaml
new file mode 100644
index 00000000..2de3e78c
--- /dev/null
+++ b/environments/storage/cinder-nfs.yaml
@@ -0,0 +1,27 @@
+# *******************************************************************
+# This file was created automatically by the sample environment
+# generator. Developers should use `tox -e genconfig` to update it.
+# Users are recommended to make changes to a copy of the file instead
+# of the original, if any customizations are needed.
+# *******************************************************************
+# title: Enable Cinder NFS Backend
+# description: |
+#   Configure and include this environment to enable the use of an NFS
+#   share as the backend for Cinder.
+parameter_defaults:
+  # Whether to enable or not the Iscsi backend for Cinder
+  # Type: boolean
+  CinderEnableIscsiBackend: False
+
+  # Whether to enable or not the NFS backend for Cinder
+  # Type: boolean
+  CinderEnableNfsBackend: True
+
+  # Mount options for NFS mounts used by Cinder NFS backend. Effective when CinderEnableNfsBackend is true.
+  # Type: string
+  CinderNfsMountOptions: ''
+
+  # NFS servers used by Cinder NFS backend. Effective when CinderEnableNfsBackend is true.
+  # Type: comma_delimited_list
+  CinderNfsServers: 192.168.122.1:/export/cinder
+
diff --git a/environments/storage/enable-ceph.yaml b/environments/storage/enable-ceph.yaml
new file mode 100644
index 00000000..c629f74b
--- /dev/null
+++ b/environments/storage/enable-ceph.yaml
@@ -0,0 +1,35 @@
+# *******************************************************************
+# This file was created automatically by the sample environment
+# generator. Developers should use `tox -e genconfig` to update it.
+# Users are recommended to make changes to a copy of the file instead
+# of the original, if any customizations are needed.
+# *******************************************************************
+# title: Enable Ceph Storage Backend
+# description: |
+#   Include this environment to enable Ceph as the backend for
+#   Cinder, Nova, Gnocchi, and Glance.
+parameter_defaults:
+  # The short name of the Cinder Backup backend to use.
+  # Type: string
+  CinderBackupBackend: rbd
+
+  # Whether to enable or not the Iscsi backend for Cinder
+  # Type: boolean
+  CinderEnableIscsiBackend: False
+
+  # Whether to enable or not the Rbd backend for Cinder
+  # Type: boolean
+  CinderEnableRbdBackend: True
+
+  # The short name of the Glance backend to use. Should be one of swift, rbd, or file
+  # Type: string
+  GlanceBackend: rbd
+
+  # The short name of the Gnocchi backend to use. Should be one of swift, rbd, or file
+  # Type: string
+  GnocchiBackend: rbd
+
+  # Whether to enable or not the Rbd backend for Nova
+  # Type: boolean
+  NovaEnableRbdBackend: True
+
diff --git a/environments/storage/external-ceph.yaml b/environments/storage/external-ceph.yaml
new file mode 100644
index 00000000..f1c9d516
--- /dev/null
+++ b/environments/storage/external-ceph.yaml
@@ -0,0 +1,78 @@
+# *******************************************************************
+# This file was created automatically by the sample environment
+# generator. Developers should use `tox -e genconfig` to update it.
+# Users are recommended to make changes to a copy of the file instead
+# of the original, if any customizations are needed.
+# *******************************************************************
+# title: Deploy Using an External Ceph Cluster
+# description: |
+#   A Heat environment file which can be used to enable the
+#   use of an externally managed Ceph cluster.
+parameter_defaults:
+  # The Ceph admin client key. Can be created with ceph-authtool --gen-print-key.
+  # Type: string
+  CephAdminKey: ''
+
+  # The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
+  # Mandatory. This parameter must be set by the user.
+  # Type: string
+  CephClientKey: <None>
+
+  # 
+  # Type: string
+  CephClientUserName: openstack
+
+  # The Ceph cluster FSID. Must be a UUID.
+  # Mandatory. This parameter must be set by the user.
+  # Type: string
+  CephClusterFSID: <None>
+
+  # List of externally managed Ceph Mon Host IPs. Only used for external Ceph deployments.
+  # Type: string
+  CephExternalMonHost: ''
+
+  # Whether to enable or not the Iscsi backend for Cinder
+  # Type: boolean
+  CinderEnableIscsiBackend: False
+
+  # Whether to enable or not the Rbd backend for Cinder
+  # Type: boolean
+  CinderEnableRbdBackend: True
+
+  # 
+  # Type: string
+  CinderRbdPoolName: volumes
+
+  # The short name of the Glance backend to use. Should be one of swift, rbd, or file
+  # Type: string
+  GlanceBackend: rbd
+
+  # 
+  # Type: string
+  GlanceRbdPoolName: images
+
+  # The short name of the Gnocchi backend to use. Should be one of swift, rbd, or file
+  # Type: string
+  GnocchiBackend: rbd
+
+  # 
+  # Type: string
+  GnocchiRbdPoolName: metrics
+
+  # Whether to enable or not the Rbd backend for Nova
+  # Type: boolean
+  NovaEnableRbdBackend: True
+
+  # 
+  # Type: string
+  NovaRbdPoolName: vms
+
+  # The default features enabled when creating a block device image. Only applies to format 2 images. Set to '1' for Jewel clients using older Ceph servers.
+  # Type: string
+  RbdDefaultFeatures: ''
+
+resource_registry:
+  OS::TripleO::Services::CephClient: OS::Heat::None
+  OS::TripleO::Services::CephExternal: ../../puppet/services/ceph-external.yaml
+  OS::TripleO::Services::CephMon: OS::Heat::None
+  OS::TripleO::Services::CephOSD: OS::Heat::None
diff --git a/environments/storage/glance-nfs.yaml b/environments/storage/glance-nfs.yaml
new file mode 100644
index 00000000..3c139306
--- /dev/null
+++ b/environments/storage/glance-nfs.yaml
@@ -0,0 +1,34 @@
+# *******************************************************************
+# This file was created automatically by the sample environment
+# generator. Developers should use `tox -e genconfig` to update it.
+# Users are recommended to make changes to a copy of the file instead
+# of the original, if any customizations are needed.
+# *******************************************************************
+# title: Enable Glance NFS Backend
+# description: |
+#   Configure and include this environment to enable the use of an NFS
+#   share as the backend for Glance.
+parameter_defaults:
+  # NFS mount options for image storage (when GlanceNfsEnabled is true)
+  # Type: string
+  GlanceNfsOptions: intr,context=system_u:object_r:glance_var_lib_t:s0
+
+  # NFS share to mount for image storage (when GlanceNfsEnabled is true)
+  # Type: string
+  GlanceNfsShare: ''
+
+  # ******************************************************
+  # Static parameters - these are values that must be
+  # included in the environment but should not be changed.
+  # ******************************************************
+  # The short name of the Glance backend to use. Should be one of swift, rbd, or file
+  # Type: string
+  GlanceBackend: file
+
+  # When using GlanceBackend 'file', mount NFS share for image storage.
+  # Type: boolean
+  GlanceNfsEnabled: True
+
+  # *********************
+  # End static parameters
+  # *********************
diff --git a/sample-env-generator/storage.yaml b/sample-env-generator/storage.yaml
new file mode 100644
index 00000000..aa0385cc
--- /dev/null
+++ b/sample-env-generator/storage.yaml
@@ -0,0 +1,133 @@
+environments:
+  -
+    name: storage/enable-ceph
+    title: Enable Ceph Storage Backend
+    files:
+      puppet/services/cinder-volume.yaml:
+        parameters:
+          - CinderEnableIscsiBackend
+          - CinderEnableRbdBackend
+      puppet/services/cinder-backup.yaml:
+        parameters:
+          - CinderBackupBackend
+      puppet/services/nova-compute.yaml:
+        parameters:
+          - NovaEnableRbdBackend
+      puppet/services/glance-api.yaml:
+        parameters:
+          - GlanceBackend
+      puppet/services/gnocchi-api.yaml:
+        parameters:
+          - GnocchiBackend
+    sample_values:
+      CinderEnableIscsiBackend: False
+      CinderEnableRbdBackend: True
+      CinderBackupBackend: rbd
+      NovaEnableRbdBackend: True
+      GlanceBackend: rbd
+      GnocchiBackend: rbd
+    description: |
+      Include this environment to enable Ceph as the backend for
+      Cinder, Nova, Gnocchi, and Glance.
+  -
+    name: storage/cinder-nfs
+    title: Enable Cinder NFS Backend
+    files:
+      puppet/services/cinder-volume.yaml:
+        parameters:
+          - CinderNfsMountOptions
+          - CinderNfsServers
+          - CinderEnableNfsBackend
+          - CinderEnableIscsiBackend
+    sample_values:
+      CinderEnableNfsBackend: True
+      CinderEnableIscsiBackend: False
+      CinderNfsServers: '192.168.122.1:/export/cinder'
+    description: |
+      Configure and include this environment to enable the use of an NFS
+      share as the backend for Cinder.
+  -
+    name: storage/glance-nfs
+    title: Enable Glance NFS Backend
+    files:
+      puppet/services/glance-api.yaml:
+        parameters:
+          - GlanceBackend
+          - GlanceNfsEnabled
+          - GlanceNfsShare
+          - GlanceNfsOptions
+    sample_values:
+      GlanceBackend: file
+      GlanceNfsEnabled: True
+    static:
+      - GlanceBackend
+      - GlanceNfsEnabled
+    description: |
+      Configure and include this environment to enable the use of an NFS
+      share as the backend for Glance.
+  -
+    name: storage/external-ceph
+    title: Deploy Using an External Ceph Cluster
+    files:
+      puppet/services/nova-compute.yaml:
+        parameters:
+          - NovaRbdPoolName
+          - NovaEnableRbdBackend
+          - CephClientUserName
+      puppet/services/cinder-volume.yaml:
+        parameters:
+          - CinderRbdPoolName
+          - CinderEnableIscsiBackend
+          - CinderEnableRbdBackend
+      puppet/services/glance-api.yaml:
+        parameters:
+          - GlanceRbdPoolName
+          - GlanceBackend
+      puppet/services/gnocchi-api.yaml:
+        parameters:
+          - GnocchiBackend
+      puppet/services/gnocchi-base.yaml:
+        parameters:
+          - GnocchiRbdPoolName
+      puppet/services/ceph-external.yaml:
+        parameters:
+          - CephClusterFSID
+          - CephClientKey
+          - CephExternalMonHost
+          - RbdDefaultFeatures
+      puppet/services/ceph-base.yaml:
+        parameters:
+          - CephAdminKey
+    sample_values:
+      CinderEnableIscsiBackend: False
+      CinderEnableRbdBackend: True
+      NovaEnableRbdBackend: True
+      GlanceBackend: rbd
+      GnocchiBackend: rbd
+      NovaRbdPoolName: vms
+      CinderRbdPoolName: volumes
+      GlanceRbdPoolName: images
+      GnocchiRbdPoolName: metrics
+      CephClientUserName: openstack
+      CephAdminKey: ''
+    description: |
+       A Heat environment file which can be used to enable the
+       use of an externally managed Ceph cluster.
+    resource_registry:
+      OS::TripleO::Services::CephExternal: ../../puppet/services/ceph-external.yaml
+      OS::TripleO::Services::CephMon: OS::Heat::None
+      OS::TripleO::Services::CephClient: OS::Heat::None
+      OS::TripleO::Services::CephOSD: OS::Heat::None
+  -
+    name: storage/cinder-netapp-config
+    title: Enable the Cinder NetApp Backend
+    description: |
+      A Heat environment file which can be used to enable a
+      a Cinder NetApp backend, configured via puppet
+    files:
+      puppet/services/cinder-backend-netapp.yaml:
+        parameters: all
+    static:
+      - CinderEnableNetappBackend
+    resource_registry:
+      OS::TripleO::ControllerExtraConfigPre: ../../puppet/extraconfig/pre_deploy/controller/cinder-netapp.yaml
diff --git a/tripleo_heat_templates/environment_generator.py b/tripleo_heat_templates/environment_generator.py
index 659a7d57..b3e327f0 100755
--- a/tripleo_heat_templates/environment_generator.py
+++ b/tripleo_heat_templates/environment_generator.py
@@ -22,7 +22,7 @@ import yaml
 
 _PARAM_FORMAT = u"""  # %(description)s
   %(mandatory)s# Type: %(type)s
-  %(name)s: %(default)s
+  %(name)s:%(default)s
 """
 _STATIC_MESSAGE_START = (
     '  # ******************************************************\n'
@@ -44,7 +44,14 @@ _FILE_HEADER = (
     )
 # Certain parameter names can't be changed, but shouldn't be shown because
 # they are never intended for direct user input.
-_PRIVATE_OVERRIDES = ['server', 'servers', 'NodeIndex']
+_PRIVATE_OVERRIDES = ['server', 'servers', 'NodeIndex', 'DefaultPasswords']
+# Hidden params are not included by default when the 'all' option is used,
+# but can be explicitly included by referencing them in sample_defaults or
+# static.  This allows us to generate sample environments using them when
+# necessary, but they won't be improperly included by accident.
+_HIDDEN_PARAMS = ['EndpointMap', 'RoleName', 'RoleParameters',
+                  'ServiceNetMap',
+                  ]
 
 
 def _create_output_dir(target_file):
@@ -64,6 +71,8 @@ def _generate_environment(input_env, parent_env=None):
     env.update(input_env)
     parameter_defaults = {}
     param_names = []
+    sample_values = env.get('sample_values', {})
+    static_names = env.get('static', [])
     for template_file, template_data in env['files'].items():
         with open(template_file) as f:
             f_data = yaml.safe_load(f)
@@ -71,6 +80,10 @@ def _generate_environment(input_env, parent_env=None):
             parameter_defaults.update(f_params)
             if template_data['parameters'] == 'all':
                 new_names = [k for k, v in f_params.items()]
+                for hidden in _HIDDEN_PARAMS:
+                    if (hidden not in (static_names + sample_values.keys()) and
+                            hidden in new_names):
+                        new_names.remove(hidden)
             else:
                 new_names = template_data['parameters']
             missing_params = [name for name in new_names
@@ -82,7 +95,6 @@ def _generate_environment(input_env, parent_env=None):
                                     env['name']))
             param_names += new_names
 
-    static_names = env.get('static', [])
     static_defaults = {k: v for k, v in parameter_defaults.items()
                        if k in param_names and
                        k in static_names
@@ -93,7 +105,8 @@ def _generate_environment(input_env, parent_env=None):
                           not k.startswith('_') and
                           k not in static_names
                           }
-    for k, v in env.get('sample_values', {}).items():
+
+    for k, v in sample_values.items():
         if k in parameter_defaults:
             parameter_defaults[k]['sample'] = v
         if k in static_defaults:
@@ -108,17 +121,18 @@ def _generate_environment(input_env, parent_env=None):
             default = '<None>'
         if value.get('sample') is not None:
             default = value['sample']
+        # We ultimately cast this to str for output anyway
+        default = str(default)
         if default == '':
             default = "''"
-        try:
-            # If the default value is something like %index%, yaml won't
-            # parse the output correctly unless we wrap it in quotes.
-            # However, not all default values can be wrapped so we need to
-            # do it conditionally.
-            if default.startswith('%'):
-                default = "'%s'" % default
-        except AttributeError:
-            pass
+        # If the default value is something like %index%, yaml won't
+        # parse the output correctly unless we wrap it in quotes.
+        # However, not all default values can be wrapped so we need to
+        # do it conditionally.
+        if default.startswith('%'):
+            default = "'%s'" % default
+        if not default.startswith('\n'):
+            default = ' ' + default
 
         values = {'name': name,
                   'type': value['type'],
diff --git a/tripleo_heat_templates/tests/test_environment_generator.py b/tripleo_heat_templates/tests/test_environment_generator.py
index d0a622da..f4c4cdbf 100644
--- a/tripleo_heat_templates/tests/test_environment_generator.py
+++ b/tripleo_heat_templates/tests/test_environment_generator.py
@@ -34,6 +34,10 @@ parameters:
     default: 42
     description: Bar description
     type: number
+  EndpointMap:
+    default: {}
+    description: Parameter that should not be included by default
+    type: json
 resources:
   # None
 '''
@@ -307,6 +311,41 @@ resource_registry:
   OS::TripleO::FakeResource: fake-filename.yaml
 ''',
           }),
+        ('basic-hidden',
+         {'template': basic_template,
+          'exception': None,
+          'input_file': '''environments:
+  -
+    name: basic
+    title: Basic Environment
+    description: Basic description
+    files:
+      foo.yaml:
+        parameters: all
+    sample_values:
+      EndpointMap: |-2
+
+            foo: bar
+''',
+          'expected_output': '''# title: Basic Environment
+# description: |
+#   Basic description
+parameter_defaults:
+  # Bar description
+  # Type: number
+  BarParam: 42
+
+  # Parameter that should not be included by default
+  # Type: json
+  EndpointMap:
+    foo: bar
+
+  # Foo description
+  # Type: string
+  FooParam: foo
+
+''',
+          }),
         ('missing-param',
          {'template': basic_template,
           'exception': RuntimeError,