Deploy Ceph in containers using ceph-ansible via external workflow

Add docker profiles to deploy Ceph in containers via ceph-ansible. This is
implemented by triggering a Mistral workflow during one of the overcloud
deployment steps, as provided by [1].

Some new service-specific parameters are available to determine the workflow to
execute and the ansible playbook to use. A new `CephAnsibleExtraConfig`
parameter can be used to provide arbitrary config variables consumed by `ceph-ansible`.

The pre-existing template params consumed up until the Pike release to
drive `puppet-ceph` continue to work and are translated, when possible, into
the equivalent `ceph-ansible` variable.

A new environment file is added to enable use of ceph-ansible;
the pre-existing puppet-ceph implementation remains unchanged and usable
for non-containerized deployments.

1. https://review.openstack.org/#/c/463324/

Change-Id: I81d44a1e198c83a4ef8b109b4eb6c611555dcdc5

4 files changed, 424 insertions, 0 deletions

diff --git a/docker/services/ceph-ansible/ceph-base.yaml b/docker/services/ceph-ansible/ceph-base.yaml
new file mode 100644
index 00000000..1468415e
--- /dev/null
+++ b/docker/services/ceph-ansible/ceph-base.yaml
@@ -0,0 +1,205 @@
+heat_template_version: pike
+
+description: >
+  Ceph base service. Shared by all Ceph services.
+
+parameters:
+  ServiceData:
+    default: {}
+    description: Dictionary packing service data
+    type: json
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  RoleName:
+    default: ''
+    description: Role name on which the service is applied
+    type: string
+  RoleParameters:
+    default: {}
+    description: Parameters specific to the role
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  CephAnsibleWorkflowName:
+    type: string
+    description: Name of the Mistral workflow to execute
+    default: tripleo.storage.v1.ceph-install
+  CephAnsiblePlaybook:
+    type: string
+    description: Path to the ceph-ansible playbook to execute
+    default: /usr/share/ceph-ansible/site-docker.yml.sample
+  CephAnsibleExtraConfig:
+    type: json
+    description: Extra vars for the ceph-ansible playbook
+    default: {}
+  CephClusterFSID:
+    type: string
+    description: The Ceph cluster FSID. Must be a UUID.
+  CephPoolDefaultPgNum:
+    description: default pg_num to use for the RBD pools
+    type: number
+    default: 32
+  CephPools:
+    description: >
+      It can be used to override settings for one of the predefined pools, or to create
+      additional ones. Example:
+      {
+        "volumes": {
+          "size": 5,
+          "pg_num": 128,
+          "pgp_num": 128
+         }
+      }
+    default: {}
+    type: json
+  CinderRbdPoolName:
+    default: volumes
+    type: string
+  CinderBackupRbdPoolName:
+    default: backups
+    type: string
+  GlanceRbdPoolName:
+    default: images
+    type: string
+  GnocchiRbdPoolName:
+    default: metrics
+    type: string
+  NovaRbdPoolName:
+    default: vms
+    type: string
+  CephClientKey:
+    description: The Ceph client key. Can be created with ceph-authtool --gen-print-key. Currently only used for external Ceph deployments to create the openstack user keyring.
+    type: string
+    hidden: true
+  CephClientUserName:
+    default: openstack
+    type: string
+  CephPoolDefaultSize:
+    description: default minimum replication for RBD copies
+    type: number
+    default: 3
+  CephIPv6:
+    default: False
+    type: boolean
+  DockerCephDaemonImage:
+    description: image
+    type: string
+    default: 'ceph/daemon:tag-build-master-jewel-centos-7'
+
+conditions:
+  custom_registry_host:
+    yaql:
+      data: {get_param: DockerCephDaemonImage}
+      expression: $.data.split('/')[0].matches('(\.|:)')
+
+outputs:
+  role_data:
+    description: Role data for the Ceph base service.
+    value:
+      service_name: ceph_base
+      upgrade_tasks: []
+      step_config: ''
+      puppet_config:
+        config_image: ''
+        config_volume: ''
+        step_config: ''
+      docker_config: {}
+      service_workflow_tasks:
+        step2:
+          - name: ceph_base_ansible_workflow
+            workflow: { get_param: CephAnsibleWorkflowName }
+            input:
+              ceph_ansible_extra_vars: {get_param: CephAnsibleExtraConfig}
+              ceph_ansible_playbook: {get_param: CephAnsiblePlaybook}
+      config_settings:
+        ceph_common_ansible_vars:
+          fsid: { get_param: CephClusterFSID }
+          docker: true
+          ceph_docker_registry:
+            if:
+            - custom_registry_host
+            - yaql:
+                expression: regex('(?:https?://)?(.*)/').split($.data)[1]
+                data: {str_split: [':', {get_param: DockerCephDaemonImage}, 0]}
+            - docker.io
+          ceph_docker_image:
+            if:
+            - custom_registry_host
+            - yaql:
+                expression: regex('(?:https?://)?(.*)/').split($.data)[2]
+                data: {str_split: [':', {get_param: DockerCephDaemonImage}, 0]}
+            - {str_split: [':', {get_param: DockerCephDaemonImage}, 0]}
+          ceph_docker_image_tag: {str_split: [':', {get_param: DockerCephDaemonImage}, 1]}
+          containerized_deployment: true
+          public_network: {get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephMonNetwork]}]}
+          cluster_network: {get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephClusterNetwork]}]}
+          user_config: true
+          ceph_stable: true
+          ceph_origin: distro
+          openstack_config: true
+          openstack_pools:
+            list_concat:
+              - repeat:
+                  template:
+                    name: <%pool%>
+                    pg_num: {get_param: CephPoolDefaultPgNum}
+                    rule_name: ""
+                  for_each:
+                    <%pool%>:
+                      - {get_param: CinderRbdPoolName}
+                      - {get_param: CinderBackupRbdPoolName}
+                      - {get_param: NovaRbdPoolName}
+                      - {get_param: GlanceRbdPoolName}
+                      - {get_param: GnocchiRbdPoolName}
+              - repeat:
+                  template:
+                    name: <%pool%>
+                    pg_num: {get_param: CephPoolDefaultPgNum}
+                    rule_name: ""
+                  for_each:
+                    <%pool%>: {get_param: CephPools}
+          openstack_keys: &openstack_keys
+          - name:
+              list_join:
+              - '.'
+              - - client
+                - {get_param: CephClientUserName}
+            key: {get_param: CephClientKey}
+            mon_cap: "allow r"
+            osd_cap:
+              str_replace:
+               template: "allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL"
+               params:
+                 NOVA_POOL: {get_param: NovaRbdPoolName}
+                 CINDER_POOL: {get_param: CinderRbdPoolName}
+                 CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName}
+                 GLANCE_POOL: {get_param: GlanceRbdPoolName}
+                 GNOCCHI_POOL: {get_param: GnocchiRbdPoolName}
+            acls:
+            - "u:glance:r--"
+            - "u:nova:r--"
+            - "u:cinder:r--"
+            - "u:gnocchi:r--"
+          keys: *openstack_keys
+          pools: []
+          ceph_conf_overrides:
+            global:
+              osd_pool_default_size: {get_param: CephPoolDefaultSize}
+              osd_pool_default_pg_num: {get_param: CephPoolDefaultPgNum}
+          ntp_service_enabled: false
+          generate_fsid: false
+          ip_version:
+            if:
+            - {get_param: CephIPv6}
+            - ipv6
+            - ipv4
diff --git a/docker/services/ceph-ansible/ceph-client.yaml b/docker/services/ceph-ansible/ceph-client.yaml
new file mode 100644
index 00000000..55d8d9da
--- /dev/null
+++ b/docker/services/ceph-ansible/ceph-client.yaml
@@ -0,0 +1,58 @@
+heat_template_version: pike
+
+description: >
+  Ceph Client service.
+
+parameters:
+  ServiceData:
+    default: {}
+    description: Dictionary packing service data
+    type: json
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  RoleName:
+    default: ''
+    description: Role name on which the service is applied
+    type: string
+  RoleParameters:
+    default: {}
+    description: Parameters specific to the role
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  CephBase:
+    type: ./ceph-base.yaml
+    properties:
+      ServiceData: {get_param: ServiceData}
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+      RoleName: {get_param: RoleName}
+      RoleParameters: {get_param: RoleParameters}
+
+outputs:
+  role_data:
+    description: Role data for the Ceph Client service.
+    value:
+      service_name: ceph_client
+      upgrade_tasks: []
+      step_config: ''
+      puppet_config:
+        config_image: ''
+        config_volume: ''
+        step_config: ''
+      docker_config: {}
+      service_workflow_tasks: {get_attr: [CephBase, role_data, service_workflow_tasks]}
+      config_settings: {}
diff --git a/docker/services/ceph-ansible/ceph-mon.yaml b/docker/services/ceph-ansible/ceph-mon.yaml
new file mode 100644
index 00000000..90149d1e
--- /dev/null
+++ b/docker/services/ceph-ansible/ceph-mon.yaml
@@ -0,0 +1,86 @@
+heat_template_version: pike
+
+description: >
+  Ceph Monitor service.
+
+parameters:
+  ServiceData:
+    default: {}
+    description: Dictionary packing service data
+    type: json
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  RoleName:
+    default: ''
+    description: Role name on which the service is applied
+    type: string
+  RoleParameters:
+    default: {}
+    description: Parameters specific to the role
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  CephMonKey:
+    description: The Ceph monitors key. Can be created with ceph-authtool --gen-print-key.
+    type: string
+    hidden: true
+  CephAdminKey:
+    default: ''
+    description: The Ceph admin client key. Can be created with ceph-authtool --gen-print-key.
+    type: string
+    hidden: true
+  CephValidationRetries:
+    type: number
+    default: 40
+    description: Number of retry attempts for Ceph validation
+  CephValidationDelay:
+    type: number
+    default: 30
+    description: Interval (in seconds) in between validation checks
+
+resources:
+  CephBase:
+    type: ./ceph-base.yaml
+    properties:
+      ServiceData: {get_param: ServiceData}
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+      RoleName: {get_param: RoleName}
+      RoleParameters: {get_param: RoleParameters}
+
+outputs:
+  role_data:
+    description: Role data for the Ceph Monitor service.
+    value:
+      service_name: ceph_mon
+      upgrade_tasks: []
+      step_config: ''
+      puppet_config:
+        config_image: ''
+        config_volume: ''
+        step_config: ''
+      docker_config: {}
+      service_workflow_tasks: {get_attr: [CephBase, role_data, service_workflow_tasks]}
+      config_settings:
+        map_merge:
+        - tripleo.ceph_mon.firewall_rules:
+            '110 ceph_mon':
+              dport:
+              - 6789
+        - ceph_mon_ansible_vars:
+            map_merge:
+            - {get_attr: [CephBase, role_data, config_settings, ceph_common_ansible_vars]}
+            - monitor_secret: {get_param: CephMonKey}
+              admin_secret: {get_param: CephAdminKey}
+              monitor_interface: br_ex
diff --git a/docker/services/ceph-ansible/ceph-osd.yaml b/docker/services/ceph-ansible/ceph-osd.yaml
new file mode 100644
index 00000000..6e0f4a60
--- /dev/null
+++ b/docker/services/ceph-ansible/ceph-osd.yaml
@@ -0,0 +1,75 @@
+heat_template_version: pike
+
+description: >
+  Ceph OSD service.
+
+parameters:
+  ServiceData:
+    default: {}
+    description: Dictionary packing service data
+    type: json
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  RoleName:
+    default: ''
+    description: Role name on which the service is applied
+    type: string
+  RoleParameters:
+    default: {}
+    description: Parameters specific to the role
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  CephAnsibleDisksConfig:
+    type: json
+    description: Disks config settings for ceph-ansible
+    default:
+      devices:
+      - /dev/vdb
+      journal_size: 512
+      journal_collocation: true
+
+resources:
+  CephBase:
+    type: ./ceph-base.yaml
+    properties:
+      ServiceData: {get_param: ServiceData}
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+      RoleName: {get_param: RoleName}
+      RoleParameters: {get_param: RoleParameters}
+
+outputs:
+  role_data:
+    description: Role data for the Ceph OSD service.
+    value:
+      service_name: ceph_osd
+      upgrade_tasks: []
+      step_config: ''
+      puppet_config:
+        config_image: ''
+        config_volume: ''
+        step_config: ''
+      docker_config: {}
+      service_workflow_tasks: {get_attr: [CephBase, role_data, service_workflow_tasks]}
+      config_settings:
+        map_merge:
+        - tripleo.ceph_osd.firewall_rules:
+            '111 ceph_osd':
+              dport:
+              - '6800-7300'
+        - ceph_osd_ansible_vars:
+            map_merge:
+            - {get_attr: [CephBase, role_data, config_settings, ceph_common_ansible_vars]}
+            - {get_param: CephAnsibleDisksConfig}
\ No newline at end of file