Switch Ceph Monitor/OSD/Client/External to composable roles

Change-Id: I1921115cb6218c7554348636c404245c79937673
Depends-On: I7ac096feb9f5655003becd79d2eea355a047c90b
Depends-On: I871ef420700e6d0ee5c1e444e019d58b3a9a45a6

21 files changed, 323 insertions, 431 deletions

diff --git a/environments/puppet-ceph-devel.yaml b/environments/puppet-ceph-devel.yaml
index a2d1100f..ad542bee 100644
--- a/environments/puppet-ceph-devel.yaml
+++ b/environments/puppet-ceph-devel.yaml
@@ -1,6 +1,11 @@
 # A Heat environment file which can be used to enable a Ceph
 # storage cluster using the controller and ceph nodes.
 # Rbd backends are enabled for Cinder, Glance, Gnocchi and Nova.
+resource_registry:
+  OS::TripleO::Services::CephMon: ../puppet/services/ceph-mon.yaml
+  OS::TripleO::Services::CephOSD: ../puppet/services/ceph-osd.yaml
+  OS::TripleO::Services::CephClient: ../puppet/services/ceph-client.yaml
+
 parameter_defaults:
   #NOTE: These ID's and keys should be regenerated for
   # a production deployment. What is here is suitable for
@@ -13,4 +18,3 @@ parameter_defaults:
   GlanceBackend: rbd
   GnocchiBackend: rbd
   CinderEnableIscsiBackend: false
-  ControllerEnableCephStorage: true
diff --git a/environments/puppet-ceph-external.yaml b/environments/puppet-ceph-external.yaml
index 865e0b98..d5e0b1b5 100644
--- a/environments/puppet-ceph-external.yaml
+++ b/environments/puppet-ceph-external.yaml
@@ -1,7 +1,7 @@
 # A Heat environment file which can be used to enable the
 # use of an externally managed Ceph cluster.
 resource_registry:
-  OS::TripleO::CephClusterConfig::SoftwareConfig: ../puppet/extraconfig/ceph/ceph-external-config.yaml
+  OS::TripleO::Services::CephExternal: ../puppet/services/ceph-external.yaml
 
 parameter_defaults:
   # NOTE: These example parameters are required when using Ceph External
diff --git a/environments/storage-environment.yaml b/environments/storage-environment.yaml
index da33acfd..2529ea14 100644
--- a/environments/storage-environment.yaml
+++ b/environments/storage-environment.yaml
@@ -1,6 +1,11 @@
 ## A Heat environment file which can be used to set up storage
 ## backends. Defaults to Ceph used as a backend for Cinder, Glance and
 ## Nova ephemeral storage.
+resource_registry:
+  OS::TripleO::Services::CephMon: ../puppet/services/ceph-mon.yaml
+  OS::TripleO::Services::CephOSD: ../puppet/services/ceph-osd.yaml
+  OS::TripleO::Services::CephClient: ../puppet/services/ceph-client.yaml
+
 parameter_defaults:
 
   #### BACKEND SELECTION ####
@@ -43,10 +48,6 @@ parameter_defaults:
 
   #### CEPH SETTINGS ####
 
-  ## Whether to deploy Ceph OSDs on the controller nodes. By default
-  ## OSDs are deployed on dedicated ceph-storage nodes only.
-  # ControllerEnableCephStorage: false
-
   ## When deploying Ceph Nodes through the oscplugin CLI, the following
   ## parameters are set automatically by the CLI. When deploying via
   ## heat stack-create or ceph on the controller nodes only,
diff --git a/overcloud-resource-registry-puppet.yaml b/overcloud-resource-registry-puppet.yaml
index cb67c7ec..1ef3660f 100644
--- a/overcloud-resource-registry-puppet.yaml
+++ b/overcloud-resource-registry-puppet.yaml
@@ -124,6 +124,10 @@ resource_registry:
 
   # services
   OS::TripleO::Services: puppet/services/services.yaml
+  OS::TripleO::Services::CephMon: OS::Heat::None
+  OS::TripleO::Services::CephOSD: OS::Heat::None
+  OS::TripleO::Services::CephClient: OS::Heat::None
+  OS::TripleO::Services::CephExternal: OS::Heat::None
   OS::TripleO::Services::CinderApi: puppet/services/cinder-api.yaml
   OS::TripleO::Services::CinderScheduler: puppet/services/cinder-scheduler.yaml
   OS::TripleO::Services::CinderVolume: puppet/services/cinder-volume.yaml
diff --git a/overcloud.yaml b/overcloud.yaml
index 1d5fd8b1..e6f23d8c 100644
--- a/overcloud.yaml
+++ b/overcloud.yaml
@@ -38,34 +38,10 @@ parameters:
     type: string
     constraints:
     - allowed_values: ['gnocchi', 'database']
-  # This has to be an UUID so for now we generate it outside the template
-  CephClusterFSID:
-    default: ''
-    type: string
-    description: The Ceph cluster FSID. Must be a UUID.
-  CephMonKey:
-    default: ''
-    description: The Ceph monitors key. Can be created with ceph-authtool --gen-print-key.
-    type: string
-    hidden: true
-  CephAdminKey:
-    default: ''
-    description: The Ceph admin client key. Can be created with ceph-authtool --gen-print-key.
-    type: string
-    hidden: true
   CinderEnableNfsBackend:
     default: false
     description: Whether to enable or not the NFS backend for Cinder
     type: boolean
-  CephClientKey:
-    default: ''
-    description: The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
-    type: string
-    hidden: true
-  CephExternalMonHost:
-    default: ''
-    type: string
-    description: List of externally managed Ceph Mon Host IPs. Only used for external Ceph deployments.
   CinderEnableRbdBackend:
     default: false
     description: Whether to enable or not the Rbd backend for Cinder
@@ -308,10 +284,6 @@ parameters:
     default: true
     description: Whether to use Galera instead of regular MariaDB.
     type: boolean
-  ControllerEnableCephStorage:
-    default: false
-    description: Whether to deploy Ceph Storage (OSD) on the Controller
-    type: boolean
   ControllerSchedulerHints:
     type: json
     description: Optional scheduler hints to pass to nova
@@ -527,6 +499,8 @@ parameters:
 
   ControllerServices:
     default:
+      - OS::TripleO::Services::CephMon
+      - OS::TripleO::Services::CephExternal
       - OS::TripleO::Services::CinderApi
       - OS::TripleO::Services::CinderScheduler
       - OS::TripleO::Services::CinderVolume
@@ -566,6 +540,8 @@ parameters:
 
   ComputeServices:
     default:
+      - OS::TripleO::Services::CephClient
+      - OS::TripleO::Services::CephExternal
       - OS::TripleO::Services::Timezone
       - OS::TripleO::Services::Ntp
       - OS::TripleO::Services::Snmp
@@ -670,6 +646,7 @@ parameters:
     default: {}
   CephStorageServices:
     default:
+      - OS::TripleO::Services::CephOSD
       - OS::TripleO::Services::Ntp
       - OS::TripleO::Services::Timezone
     description: A list of service resources (configured in the Heat
@@ -824,7 +801,6 @@ resources:
           ManageFirewall: {get_param: ManageFirewall}
           PurgeFirewallRules: {get_param: PurgeFirewallRules}
           EnableGalera: {get_param: EnableGalera}
-          EnableCephStorage: {get_param: ControllerEnableCephStorage}
           ExtraConfig: {get_param: ExtraConfig}
           FencingConfig: {get_param: FencingConfig}
           Flavor: {get_param: OvercloudControlFlavor}
@@ -1105,6 +1081,8 @@ resources:
       keystone_admin_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}]}
       sahara_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
       ironic_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, IronicApiNetwork]}]}
+      ceph_mon_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
+      ceph_mon_node_names: {get_attr: [Controller, hostname]}
       DeployIdentifier: {get_param: DeployIdentifier}
       UpdateIdentifier: {get_param: UpdateIdentifier}
 
@@ -1289,15 +1267,6 @@ resources:
 
   CephClusterConfig:
     type: OS::TripleO::CephClusterConfig::SoftwareConfig
-    properties:
-      ceph_storage_count: {get_param: CephStorageCount}
-      ceph_fsid: {get_param: CephClusterFSID}
-      ceph_mon_key: {get_param: CephMonKey}
-      ceph_admin_key: {get_param: CephAdminKey}
-      ceph_client_key: {get_param: CephClientKey}
-      ceph_external_mon_ips: {get_param: CephExternalMonHost}
-      ceph_mon_names: {get_attr: [Controller, hostname]}
-      ceph_mon_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
 
   ControllerAllNodesDeployment:
     type: OS::Heat::StructuredDeployments
diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml
index a53d8981..7602d8d0 100644
--- a/puppet/all-nodes-config.yaml
+++ b/puppet/all-nodes-config.yaml
@@ -58,7 +58,10 @@ parameters:
     type: comma_delimited_list
   ironic_api_node_ips:
     type: comma_delimited_list
-
+  ceph_mon_node_ips:
+    type: comma_delimited_list
+  ceph_mon_node_names:
+    type: comma_delimited_list
   DeployIdentifier:
     type: string
     description: >
@@ -305,6 +308,22 @@ resources:
                         - "','"
                         - {get_param: ironic_api_node_ips}
 
+                tripleo::profile::base::ceph::ceph_mon_initial_members:
+                  list_join:
+                  - ','
+                  - {get_param: ceph_mon_node_names}
+                tripleo::profile::base::ceph::ceph_mon_host:
+                  list_join:
+                  - ','
+                  - {get_param: ceph_mon_node_ips}
+                tripleo::profile::base::ceph::ceph_mon_host_v6:
+                  str_replace:
+                    template: "'[IPS_LIST]'"
+                    params:
+                      IPS_LIST:
+                        list_join:
+                        - '],['
+                        - {get_param: ceph_mon_node_ips}
                 # NOTE(gfidente): interpolation with %{} in the
                 # hieradata file can't be used as it returns string
                 ceilometer::rabbit_hosts: *rabbit_nodes_array
diff --git a/puppet/ceph-cluster-config.yaml b/puppet/ceph-cluster-config.yaml
index 6beb751f..2e329989 100644
--- a/puppet/ceph-cluster-config.yaml
+++ b/puppet/ceph-cluster-config.yaml
@@ -2,52 +2,15 @@ heat_template_version: 2015-04-30
 description: 'Ceph Cluster config data for Puppet'
 
 parameters:
-  ceph_storage_count:
-    default: 0
-    type: number
-    description: Number of Ceph storage nodes. Used to enable/disable managed Ceph installation.
-  ceph_external_mon_ips:
-    default: ''
-    type: string
-    description: List of external Ceph Mon host IPs.
-  ceph_client_key:
-    default: ''
-    type: string
-    description: Ceph key used to create the client user keyring.
-  ceph_fsid:
-    default: ''
-    type: string
-  ceph_admin_key:
-    default: ''
-    type: string
-  ceph_mon_key:
-    default: ''
-    type: string
-  ceph_mon_names:
-    type: comma_delimited_list
-  ceph_mon_ips:
-    type: comma_delimited_list
   NovaRbdPoolName:
     default: vms
     type: string
-  CinderRbdPoolName:
-    default: volumes
-    type: string
-  CinderBackupRbdPoolName:
-    default: backups
-    type: string
-  GlanceRbdPoolName:
-    default: images
-    type: string
   GnocchiRbdPoolName:
     default: metrics
     type: string
   CephClientUserName:
     default: openstack
     type: string
-  CephIPv6:
-    default: False
-    type: boolean
 
 resources:
   CephClusterConfigImpl:
@@ -59,66 +22,10 @@ resources:
           datafiles:
             ceph_cluster:
               mapped_data:
-                ceph_ipv6: {get_param: CephIPv6}
-                ceph_storage_count: {get_param: ceph_storage_count}
-                ceph_mon_initial_members:
-                  list_join:
-                  - ','
-                  - {get_param: ceph_mon_names}
-                ceph_mon_host:
-                  list_join:
-                  - ','
-                  - {get_param: ceph_mon_ips}
-                ceph_mon_host_v6:
-                  str_replace:
-                    template: "'[IPS_LIST]'"
-                    params:
-                      IPS_LIST:
-                        list_join:
-                        - '],['
-                        - {get_param: ceph_mon_ips}
-                ceph::profile::params::ms_bind_ipv6: {get_param: CephIPv6}
-                ceph::profile::params::fsid: {get_param: ceph_fsid}
-                ceph::profile::params::mon_key: {get_param: ceph_mon_key}
-                # We should use a separated key for the non-admin clients
-                ceph::profile::params::client_keys:
-                  str_replace:
-                    template: "{
-                      client.admin: {
-                        secret: 'ADMIN_KEY',
-                        mode: '0600',
-                        cap_mon: 'allow *',
-                        cap_osd: 'allow *',
-                        cap_mds: 'allow *'
-                      },
-                      client.bootstrap-osd: {
-                        secret: 'ADMIN_KEY',
-                        keyring_path: '/var/lib/ceph/bootstrap-osd/ceph.keyring',
-                        cap_mon: 'allow profile bootstrap-osd'
-                      },
-                      client.CLIENT_USER: {
-                        secret: 'CLIENT_KEY',
-                        mode: '0644',
-                        cap_mon: 'allow r',
-                        cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL'
-                      }
-                    }"
-                    params:
-                      CLIENT_USER: {get_param: CephClientUserName}
-                      CLIENT_KEY: {get_param: ceph_client_key}
-                      ADMIN_KEY: {get_param: ceph_admin_key}
-                      NOVA_POOL: {get_param: NovaRbdPoolName}
-                      CINDER_POOL: {get_param: CinderRbdPoolName}
-                      CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName}
-                      GLANCE_POOL: {get_param: GlanceRbdPoolName}
-                      GNOCCHI_POOL: {get_param: GnocchiRbdPoolName}
                 nova::compute::rbd::libvirt_images_rbd_pool: {get_param: NovaRbdPoolName}
-                tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name: {get_param: CinderRbdPoolName}
-                glance::backend::rbd::rbd_store_pool: {get_param: GlanceRbdPoolName}
                 gnocchi::storage::ceph::ceph_pool: {get_param: GnocchiRbdPoolName}
                 gnocchi::storage::ceph::ceph_username: {get_param: CephClientUserName}
                 nova::compute::rbd::libvirt_rbd_user: {get_param: CephClientUserName}
-                glance::backend::rbd::rbd_store_user: {get_param: CephClientUserName}
                 nova::compute::rbd::rbd_keyring:
                   list_join:
                   - '.'
@@ -131,13 +38,6 @@ resources:
                     - 'client'
                     - {get_param: CephClientUserName}
                     - 'keyring'
-                tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
-                ceph_pools:
-                  - {get_param: CinderRbdPoolName}
-                  - {get_param: CinderBackupRbdPoolName}
-                  - {get_param: NovaRbdPoolName}
-                  - {get_param: GlanceRbdPoolName}
-                  - {get_param: GnocchiRbdPoolName}
 
 outputs:
   config_id:
diff --git a/puppet/ceph-storage.yaml b/puppet/ceph-storage.yaml
index 28493964..b242526a 100644
--- a/puppet/ceph-storage.yaml
+++ b/puppet/ceph-storage.yaml
@@ -243,6 +243,7 @@ resources:
             - service_configs
             - ceph_cluster # provided by CephClusterConfig
             - ceph
+            - all_nodes # provided by allNodesConfig
             - '"%{::osfamily}"'
             - common
             - network
diff --git a/puppet/controller.yaml b/puppet/controller.yaml
index e9833b4c..2d15bbfa 100644
--- a/puppet/controller.yaml
+++ b/puppet/controller.yaml
@@ -83,10 +83,6 @@ parameters:
     default: true
     description: Whether to deploy a LoadBalancer on the Controller
     type: boolean
-  EnableCephStorage:
-    default: false
-    description: Whether to deploy Ceph Storage (OSD) on the Controller
-    type: boolean
   ExtraConfig:
     default: {}
     description: |
@@ -575,7 +571,6 @@ resources:
         enable_fencing: {get_param: EnableFencing}
         enable_galera: {get_param: EnableGalera}
         enable_load_balancer: {get_param: EnableLoadBalancer}
-        enable_ceph_storage: {get_param: EnableCephStorage}
         manage_firewall: {get_param: ManageFirewall}
         purge_firewall_rules: {get_param: PurgeFirewallRules}
         mysql_innodb_buffer_pool_size: {get_param: MysqlInnodbBufferPoolSize}
@@ -851,7 +846,6 @@ resources:
                 # MySQL
                 admin_password: {get_input: admin_password}
                 enable_galera: {get_input: enable_galera}
-                enable_ceph_storage: {get_input: enable_ceph_storage}
                 mysql_innodb_buffer_pool_size: {get_input: mysql_innodb_buffer_pool_size}
                 mysql_max_connections: {get_input: mysql_max_connections}
                 mysql::server::root_password: {get_input: mysql_root_password}
diff --git a/puppet/extraconfig/ceph/ceph-external-config.yaml b/puppet/extraconfig/ceph/ceph-external-config.yaml
deleted file mode 100644
index 7c05a5b9..00000000
--- a/puppet/extraconfig/ceph/ceph-external-config.yaml
+++ /dev/null
@@ -1,120 +0,0 @@
-heat_template_version: 2015-04-30
-description: 'Configure parameters for an external Ceph cluster via Puppet.'
-
-parameters:
-  ceph_storage_count:
-    default: 0
-    type: number
-    description: Number of Ceph storage nodes. Used to enable/disable managed Ceph installation.
-  ceph_external_mon_ips:
-    default: ''
-    type: string
-    description: List of external Ceph Mon host IPs.
-  ceph_client_key:
-    default: ''
-    type: string
-    description: Ceph key used to create the 'openstack' user keyring.
-  ceph_fsid:
-    default: ''
-    type: string
-  # The following parameters are unused for external Ceph clusters and
-  # are here and exist for compatibility
-  ceph_admin_key:
-    default: ''
-    type: string
-  ceph_mon_key:
-    default: ''
-    type: string
-  ceph_mon_names:
-    type: comma_delimited_list
-  ceph_mon_ips:
-    type: comma_delimited_list
-  NovaRbdPoolName:
-    default: vms
-    type: string
-  CinderRbdPoolName:
-    default: volumes
-    type: string
-  CinderBackupRbdPoolName:
-    default: backups
-    type: string
-  GlanceRbdPoolName:
-    default: images
-    type: string
-  GnocchiRbdPoolName:
-    default: metrics
-    type: string
-  CephClientUserName:
-    default: openstack
-    type: string
-  CephIPv6:
-    default: False
-    type: boolean
-
-resources:
-  CephClusterConfigImpl:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: os-apply-config
-      config:
-        hiera:
-          datafiles:
-            ceph_cluster:
-              mapped_data:
-                ceph_storage_count: {get_param: ceph_storage_count}
-                enable_external_ceph: true
-                ceph_ipv6: {get_param: CephIPv6}
-                ceph_mon_host: {get_param: ceph_external_mon_ips}
-                ceph_mon_host_v6: {get_param: ceph_external_mon_ips}
-                ceph::profile::params::fsid: {get_param: ceph_fsid}
-                ceph::profile::params::client_keys:
-                  str_replace:
-                    template: "{
-                      client.CLIENT_USER: {
-                        secret: 'CLIENT_KEY',
-                        mode: '0644',
-                        cap_mon: 'allow r',
-                        cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL'
-                      }
-                    }"
-                    params:
-                      CLIENT_USER: {get_param: CephClientUserName}
-                      CLIENT_KEY: {get_param: ceph_client_key}
-                      NOVA_POOL: {get_param: NovaRbdPoolName}
-                      CINDER_POOL: {get_param: CinderRbdPoolName}
-                      CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName}
-                      GLANCE_POOL: {get_param: GlanceRbdPoolName}
-                      GNOCCHI_POOL: {get_param: GnocchiRbdPoolName}
-                ceph::profile::params::ms_bind_ipv6: {get_param: CephIPv6}
-                nova::compute::rbd::libvirt_images_rbd_pool: {get_param: NovaRbdPoolName}
-                tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name: {get_param: CinderRbdPoolName}
-                glance::backend::rbd::rbd_store_pool: {get_param: GlanceRbdPoolName}
-                gnocchi::storage::ceph::ceph_pool: {get_param: GnocchiRbdPoolName}
-                gnocchi::storage::ceph::ceph_username: {get_param: CephClientUserName}
-                nova::compute::rbd::libvirt_rbd_user: {get_param: CephClientUserName}
-                glance::backend::rbd::rbd_store_user: {get_param: CephClientUserName}
-                nova::compute::rbd::rbd_keyring:
-                  list_join:
-                  - '.'
-                  - - 'client'
-                    - {get_param: CephClientUserName}
-                gnocchi::storage::ceph::ceph_keyring:
-                  list_join:
-                  - '.'
-                  - - '/etc/ceph/ceph'
-                    - 'client'
-                    - {get_param: CephClientUserName}
-                    - 'keyring'
-                tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
-                ceph_pools:
-                  - {get_param: CinderRbdPoolName}
-                  - {get_param: CinderBackupRbdPoolName}
-                  - {get_param: NovaRbdPoolName}
-                  - {get_param: GlanceRbdPoolName}
-                  - {get_param: GnocchiRbdPoolName}
-
-outputs:
-  config_id:
-    description: The ID of the CephClusterConfigImpl resource.
-    value:
-      {get_resource: CephClusterConfigImpl}
diff --git a/puppet/manifests/overcloud_cephstorage.pp b/puppet/manifests/overcloud_cephstorage.pp
index 2169002e..af6b0960 100644
--- a/puppet/manifests/overcloud_cephstorage.pp
+++ b/puppet/manifests/overcloud_cephstorage.pp
@@ -24,33 +24,7 @@ if hiera('step') >= 1 {
 
 }
 
-if hiera('step') >= 3 {
-  if str2bool(hiera('ceph_osd_selinux_permissive', true)) {
-    exec { 'set selinux to permissive on boot':
-      command => "sed -ie 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config",
-      onlyif  => "test -f /etc/selinux/config && ! grep '^SELINUX=permissive' /etc/selinux/config",
-      path    => ['/usr/bin', '/usr/sbin'],
-    }
-
-    exec { 'set selinux to permissive':
-      command => 'setenforce 0',
-      onlyif  => "which setenforce && getenforce | grep -i 'enforcing'",
-      path    => ['/usr/bin', '/usr/sbin'],
-    } -> Class['ceph::profile::osd']
-  }
-
-  if str2bool(hiera('ceph_ipv6', false)) {
-    $mon_host = hiera('ceph_mon_host_v6')
-  } else {
-    $mon_host = hiera('ceph_mon_host')
-  }
-  class { '::ceph::profile::params':
-    mon_host            => $mon_host,
-  }
-  include ::ceph::conf
-  include ::ceph::profile::client
-  include ::ceph::profile::osd
-
+if hiera('step') >= 4 {
   hiera_include('ceph_classes')
 }
 
diff --git a/puppet/manifests/overcloud_compute.pp b/puppet/manifests/overcloud_compute.pp
index 92039bc6..b8e267fc 100644
--- a/puppet/manifests/overcloud_compute.pp
+++ b/puppet/manifests/overcloud_compute.pp
@@ -32,21 +32,6 @@ if hiera('step') >= 4 {
     ensure => present,
   }
 
-  $rbd_ephemeral_storage = hiera('nova::compute::rbd::ephemeral_storage', false)
-  $rbd_persistent_storage = hiera('rbd_persistent_storage', false)
-  if $rbd_ephemeral_storage or $rbd_persistent_storage {
-    if str2bool(hiera('ceph_ipv6', false)) {
-      $mon_host = hiera('ceph_mon_host_v6')
-    } else {
-      $mon_host = hiera('ceph_mon_host')
-    }
-    class { '::ceph::profile::params':
-      mon_host            => $mon_host,
-    }
-    include ::ceph::conf
-    include ::ceph::profile::client
-  }
-
   nova_config {
     'DEFAULT/my_ip': value => $ipaddress;
     'DEFAULT/linuxnet_interface_driver': value => 'nova.network.linux_net.LinuxOVSInterfaceDriver';
diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index d8166f20..72c31a31 100644
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -79,55 +79,6 @@ if hiera('step') >= 2 {
   }
   include ::aodh::db::mysql
 
-  $enable_ceph = hiera('ceph_storage_count', 0) > 0 or hiera('enable_ceph_storage', false)
-
-  if $enable_ceph {
-    $mon_initial_members = downcase(hiera('ceph_mon_initial_members'))
-    if str2bool(hiera('ceph_ipv6', false)) {
-      $mon_host = hiera('ceph_mon_host_v6')
-    } else {
-      $mon_host = hiera('ceph_mon_host')
-    }
-    class { '::ceph::profile::params':
-      mon_initial_members => $mon_initial_members,
-      mon_host            => $mon_host,
-    }
-    include ::ceph::conf
-    include ::ceph::profile::mon
-  }
-
-  if str2bool(hiera('enable_ceph_storage', false)) {
-    if str2bool(hiera('ceph_osd_selinux_permissive', true)) {
-      exec { 'set selinux to permissive on boot':
-        command => "sed -ie 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config",
-        onlyif  => "test -f /etc/selinux/config && ! grep '^SELINUX=permissive' /etc/selinux/config",
-        path    => ['/usr/bin', '/usr/sbin'],
-      }
-
-      exec { 'set selinux to permissive':
-        command => 'setenforce 0',
-        onlyif  => "which setenforce && getenforce | grep -i 'enforcing'",
-        path    => ['/usr/bin', '/usr/sbin'],
-      } -> Class['ceph::profile::osd']
-    }
-
-    include ::ceph::conf
-    include ::ceph::profile::osd
-  }
-
-  if str2bool(hiera('enable_external_ceph', false)) {
-    if str2bool(hiera('ceph_ipv6', false)) {
-      $mon_host = hiera('ceph_mon_host_v6')
-    } else {
-      $mon_host = hiera('ceph_mon_host')
-    }
-    class { '::ceph::profile::params':
-      mon_host            => $mon_host,
-    }
-    include ::ceph::conf
-    include ::ceph::profile::client
-  }
-
 } #END STEP 2
 
 if hiera('step') >= 4 {
@@ -204,15 +155,6 @@ if hiera('step') >= 4 {
     }
   }
 
-  if $enable_ceph {
-    $ceph_pools = hiera('ceph_pools')
-    ceph::pool { $ceph_pools :
-      pg_num  => hiera('ceph::profile::params::osd_pool_default_pg_num'),
-      pgp_num => hiera('ceph::profile::params::osd_pool_default_pgp_num'),
-      size    => hiera('ceph::profile::params::osd_pool_default_size'),
-    }
-  }
-
   # Ceilometer
   $ceilometer_backend = downcase(hiera('ceilometer_backend'))
   case $ceilometer_backend {
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index 9f636ddd..7205002a 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -262,57 +262,6 @@ if hiera('step') >= 2 {
       }
   }
 
-  # Ceph
-  $enable_ceph = hiera('ceph_storage_count', 0) > 0 or hiera('enable_ceph_storage', false)
-
-  if $enable_ceph {
-    $mon_initial_members = downcase(hiera('ceph_mon_initial_members'))
-    if str2bool(hiera('ceph_ipv6', false)) {
-      $mon_host = hiera('ceph_mon_host_v6')
-    } else {
-      $mon_host = hiera('ceph_mon_host')
-    }
-    class { '::ceph::profile::params':
-      mon_initial_members => $mon_initial_members,
-      mon_host            => $mon_host,
-    }
-    include ::ceph::conf
-    include ::ceph::profile::mon
-  }
-
-  if str2bool(hiera('enable_ceph_storage', false)) {
-    if str2bool(hiera('ceph_osd_selinux_permissive', true)) {
-      exec { 'set selinux to permissive on boot':
-        command => "sed -ie 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config",
-        onlyif  => "test -f /etc/selinux/config && ! grep '^SELINUX=permissive' /etc/selinux/config",
-        path    => ['/usr/bin', '/usr/sbin'],
-      }
-
-      exec { 'set selinux to permissive':
-        command => 'setenforce 0',
-        onlyif  => "which setenforce && getenforce | grep -i 'enforcing'",
-        path    => ['/usr/bin', '/usr/sbin'],
-      } -> Class['ceph::profile::osd']
-    }
-
-    include ::ceph::conf
-    include ::ceph::profile::osd
-  }
-
-  if str2bool(hiera('enable_external_ceph', false)) {
-    if str2bool(hiera('ceph_ipv6', false)) {
-      $mon_host = hiera('ceph_mon_host_v6')
-    } else {
-      $mon_host = hiera('ceph_mon_host')
-    }
-    class { '::ceph::profile::params':
-      mon_host            => $mon_host,
-    }
-    include ::ceph::conf
-    include ::ceph::profile::client
-  }
-
-
 } #END STEP 2
 
 if hiera('step') >= 4 or ( hiera('step') >= 3 and $sync_db ) {
@@ -398,15 +347,6 @@ MYSQL_HOST=localhost\n",
     }
   }
 
-  if $enable_ceph {
-    $ceph_pools = hiera('ceph_pools')
-    ceph::pool { $ceph_pools :
-      pg_num  => hiera('ceph::profile::params::osd_pool_default_pg_num'),
-      pgp_num => hiera('ceph::profile::params::osd_pool_default_pgp_num'),
-      size    => hiera('ceph::profile::params::osd_pool_default_size'),
-    }
-  }
-
   # Ceilometer
   case downcase(hiera('ceilometer_backend')) {
     /mysql/: {
diff --git a/puppet/services/ceph-base.yaml b/puppet/services/ceph-base.yaml
new file mode 100644
index 00000000..065901b8
--- /dev/null
+++ b/puppet/services/ceph-base.yaml
@@ -0,0 +1,94 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Ceph base service. Shared by all Ceph services.
+
+parameters:
+  CephAdminKey:
+    default: ''
+    description: The Ceph admin client key. Can be created with ceph-authtool --gen-print-key.
+    type: string
+    hidden: true
+  CephClientKey:
+    default: ''
+    description: The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
+    type: string
+    hidden: true
+  CephClientUserName:
+    default: openstack
+    type: string
+  CephClusterFSID:
+    default: ''
+    type: string
+    description: The Ceph cluster FSID. Must be a UUID.
+  CephIPv6:
+    default: False
+    type: boolean
+  CinderRbdPoolName:
+    default: volumes
+    type: string
+  CinderBackupRbdPoolName:
+    default: backups
+    type: string
+  GlanceRbdPoolName:
+    default: images
+    type: string
+  GnocchiRbdPoolName:
+    default: metrics
+    type: string
+  NovaRbdPoolName:
+    default: vms
+    type: string
+  # DEPRECATED options for compatibility with overcloud.yaml
+  # This should be removed and manipulation of the ControllerServices list
+  # used instead, but we need client support for that first
+  ControllerEnableCephStorage:
+    default: false
+    description: Whether to deploy Ceph Storage (OSD) on the Controller
+    type: boolean
+
+parameter_groups:
+- label: deprecated
+  description: Do not use deprecated params, they will be removed.
+  parameters:
+  - ControllerEnableCephStorage
+
+outputs:
+  role_data:
+    description: Role data for the Ceph base service.
+    value:
+      config_settings:
+        tripleo::profile::base::ceph::ceph_ipv6: {get_param: CephIPv6}
+        tripleo::profile::base::ceph::enable_ceph_storage: {get_param: ControllerEnableCephStorage}
+        ceph::profile::params::fsid: {get_param: CephClusterFSID}
+        ceph::profile::params::client_keys:
+          str_replace:
+            template: "{
+              client.admin: {
+                secret: 'ADMIN_KEY',
+                mode: '0600',
+                cap_mon: 'allow *',
+                cap_osd: 'allow *',
+                cap_mds: 'allow *'
+              },
+              client.bootstrap-osd: {
+                secret: 'ADMIN_KEY',
+                keyring_path: '/var/lib/ceph/bootstrap-osd/ceph.keyring',
+                cap_mon: 'allow profile bootstrap-osd'
+              },
+              client.CLIENT_USER: {
+                secret: 'CLIENT_KEY',
+                mode: '0644',
+                cap_mon: 'allow r',
+                cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL'
+              }
+            }"
+            params:
+              CLIENT_USER: {get_param: CephClientUserName}
+              CLIENT_KEY: {get_param: CephClientKey}
+              ADMIN_KEY: {get_param: CephAdminKey}
+              NOVA_POOL: {get_param: NovaRbdPoolName}
+              CINDER_POOL: {get_param: CinderRbdPoolName}
+              CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName}
+              GLANCE_POOL: {get_param: GlanceRbdPoolName}
+              GNOCCHI_POOL: {get_param: GnocchiRbdPoolName}
diff --git a/puppet/services/ceph-client.yaml b/puppet/services/ceph-client.yaml
new file mode 100644
index 00000000..ca920a5f
--- /dev/null
+++ b/puppet/services/ceph-client.yaml
@@ -0,0 +1,24 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Ceph Client service.
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  CephBase:
+    type: ./ceph-base.yaml
+
+outputs:
+  role_data:
+    description: Role data for the Cinder OSD service.
+    value:
+      config_settings:
+        get_attr: [CephBase, role_data, config_settings]
+      step_config: |
+        include ::tripleo::profile::base::ceph::client
diff --git a/puppet/services/ceph-external.yaml b/puppet/services/ceph-external.yaml
new file mode 100644
index 00000000..4522f416
--- /dev/null
+++ b/puppet/services/ceph-external.yaml
@@ -0,0 +1,65 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Ceph External service.
+
+parameters:
+  CephClientKey:
+    default: ''
+    description: The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
+    type: string
+    hidden: true
+  CephClientUserName:
+    default: openstack
+    type: string
+  CephClusterFSID:
+    default: ''
+    type: string
+    description: The Ceph cluster FSID. Must be a UUID.
+  CephExternalMonHost:
+    default: ''
+    type: string
+    description: List of externally managed Ceph Mon Host IPs. Only used for external Ceph deployments.
+  CinderRbdPoolName:
+    default: volumes
+    type: string
+  CinderBackupRbdPoolName:
+    default: backups
+    type: string
+  GlanceRbdPoolName:
+    default: images
+    type: string
+  GnocchiRbdPoolName:
+    default: metrics
+    type: string
+  NovaRbdPoolName:
+    default: vms
+    type: string
+
+outputs:
+  role_data:
+    description: Role data for the Ceph External service.
+    value:
+      config_settings:
+        tripleo::profile::base::ceph::ceph_mon_host: {get_param: CephExternalMonHost}
+        ceph::profile::params::fsid: {get_param: CephClusterFSID}
+        ceph::profile::params::client_keys:
+          str_replace:
+            template: "{
+              client.CLIENT_USER: {
+                secret: 'CLIENT_KEY',
+                mode: '0644',
+                cap_mon: 'allow r',
+                cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL'
+              }
+            }"
+            params:
+              CLIENT_USER: {get_param: CephClientUserName}
+              CLIENT_KEY: {get_param: CephClientKey}
+              NOVA_POOL: {get_param: NovaRbdPoolName}
+              CINDER_POOL: {get_param: CinderRbdPoolName}
+              CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName}
+              GLANCE_POOL: {get_param: GlanceRbdPoolName}
+              GNOCCHI_POOL: {get_param: GnocchiRbdPoolName}
+      step_config: |
+        include ::tripleo::profile::base::ceph::client
diff --git a/puppet/services/ceph-mon.yaml b/puppet/services/ceph-mon.yaml
new file mode 100644
index 00000000..d6e3aa70
--- /dev/null
+++ b/puppet/services/ceph-mon.yaml
@@ -0,0 +1,56 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Ceph Monitor service.
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  CephIPv6:
+    default: False
+    type: boolean
+  CephMonKey:
+    default: ''
+    description: The Ceph monitors key. Can be created with ceph-authtool --gen-print-key.
+    type: string
+    hidden: true
+  CinderRbdPoolName:
+    default: volumes
+    type: string
+  CinderBackupRbdPoolName:
+    default: backups
+    type: string
+  GlanceRbdPoolName:
+    default: images
+    type: string
+  GnocchiRbdPoolName:
+    default: metrics
+    type: string
+  NovaRbdPoolName:
+    default: vms
+    type: string
+
+resources:
+  CephBase:
+    type: ./ceph-base.yaml
+
+outputs:
+  role_data:
+    description: Role data for the Ceph Monitor service.
+    value:
+      config_settings:
+        map_merge:
+          - get_attr: [CephBase, role_data, config_settings]
+          - ceph::profile::params::ms_bind_ipv6: {get_param: CephIPv6}
+            ceph::profile::params::mon_key: {get_param: CephMonKey}
+            tripleo::profile::base::ceph::mon::ceph_pools:
+              - {get_param: CinderRbdPoolName}
+              - {get_param: CinderBackupRbdPoolName}
+              - {get_param: NovaRbdPoolName}
+              - {get_param: GlanceRbdPoolName}
+              - {get_param: GnocchiRbdPoolName}
+      step_config: |
+        include ::tripleo::profile::base::ceph::mon
diff --git a/puppet/services/ceph-osd.yaml b/puppet/services/ceph-osd.yaml
new file mode 100644
index 00000000..24f60283
--- /dev/null
+++ b/puppet/services/ceph-osd.yaml
@@ -0,0 +1,24 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Ceph OSD service.
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  CephBase:
+    type: ./ceph-base.yaml
+
+outputs:
+  role_data:
+    description: Role data for the Cinder OSD service.
+    value:
+      config_settings:
+        get_attr: [CephBase, role_data, config_settings]
+      step_config: |
+        include ::tripleo::profile::base::ceph::osd
diff --git a/puppet/services/cinder-volume.yaml b/puppet/services/cinder-volume.yaml
index d28f40e6..41f3827d 100644
--- a/puppet/services/cinder-volume.yaml
+++ b/puppet/services/cinder-volume.yaml
@@ -36,6 +36,12 @@ parameters:
       NFS servers used by Cinder NFS backend. Effective when
       CinderEnableNfsBackend is true.
     type: comma_delimited_list
+  CinderRbdPoolName:
+    default: volumes
+    type: string
+  CephClientUserName:
+    default: openstack
+    type: string
   EndpointMap:
     default: {}
     description: Mapping of service endpoint -> protocol. Typically set
@@ -67,5 +73,7 @@ outputs:
                   SERVERS: {get_param: CinderNfsServers}
             tripleo::profile::base::cinder::volume::iscsi::cinder_lvm_loop_device_size: {get_param: CinderLVMLoopDeviceSize}
             tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: {get_param: CinderISCSIHelper}
+            tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name: {get_param: CinderRbdPoolName}
+            tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
       step_config: |
         include ::tripleo::profile::base::cinder::volume
diff --git a/puppet/services/glance-api.yaml b/puppet/services/glance-api.yaml
index 89e6ee0f..f1f98a8e 100644
--- a/puppet/services/glance-api.yaml
+++ b/puppet/services/glance-api.yaml
@@ -9,6 +9,9 @@ parameters:
     description: Mapping of service endpoint -> protocol. Typically set
                  via parameter_defaults in the resource registry.
     type: json
+  CephClientUserName:
+    default: openstack
+    type: string
   Debug:
     default: ''
     description: Set to True to enable debugging on all services.
@@ -36,6 +39,9 @@ parameters:
     default: 0
     description: Number of workers for Glance service.
     type: number
+  GlanceRbdPoolName:
+    default: images
+    type: string
   RabbitPassword:
     description: The password for RabbitMQ
     type: string
@@ -85,6 +91,8 @@ outputs:
         glance::backend::swift::swift_store_auth_address: {get_param: [EndpointMap, KeystoneInternal, uri] }
         glance::backend::swift::swift_store_user: service:glance
         glance::backend::swift::swift_store_key: {get_param: GlancePassword}
+        glance::backend::rbd::rbd_store_pool: {get_param: GlanceRbdPoolName}
+        glance::backend::rbd::rbd_store_user: {get_param: CephClientUserName}
         glance_backend: {get_param: GlanceBackend}
         glance::db::mysql::password: {get_param: GlancePassword}
         glance::notify::rabbitmq::rabbit_userid: {get_param: RabbitUserName}