Integrate ceph in XCI

Openstack ansible support to deploy ceph.
The purpose of this patch is to configure the ceph,
just like we configure other openstack components.

The default is to not deploy ceph.
If you want to deploy ceph you just need to
export XCI_CEPH_ENABLED=true before running xci-deploy.sh.
When deployed successfully, the openstack storage will use ceph.

Change-Id: Ifd8d16fdce2914b6316842e72bbfd93228ea059d
Signed-off-by: wutianwei <wutianwei1@huawei.com>

16 files changed, 259 insertions, 0 deletions

diff --git a/xci/config/user-vars b/xci/config/user-vars
index 7c99ee9d..e9980e34 100755
--- a/xci/config/user-vars
+++ b/xci/config/user-vars
@@ -21,6 +21,7 @@
 #   export XCI_FLAVOR="ha"
 #-------------------------------------------------------------------------------
 export XCI_FLAVOR=${XCI_FLAVOR:-aio}
+export XCI_CEPH_ENABLED=${XCI_CEPH_ENABLED:-false}
 
 #-------------------------------------------------------------------------------
 # Set Paths to where git repositories of XCI Components will be cloned
diff --git a/xci/file/aio/configure-opnfvhost.yml b/xci/file/aio/configure-opnfvhost.yml
index b60c746b..41ae0b3b 100644
--- a/xci/file/aio/configure-opnfvhost.yml
+++ b/xci/file/aio/configure-opnfvhost.yml
@@ -23,3 +23,5 @@
       command: "/bin/bash ./scripts/bootstrap-aio.sh"
       args:
         chdir: "{{OPENSTACK_OSA_PATH}}"
+      environment:
+        SCENARIO: "{{ (XCI_CEPH_ENABLED == 'true') | ternary('ceph', 'aio') }}"
diff --git a/xci/file/ha/ceph.yml b/xci/file/ha/ceph.yml
new file mode 100644
index 00000000..1567c492
--- /dev/null
+++ b/xci/file/ha/ceph.yml
@@ -0,0 +1,15 @@
+# The infra nodes where the Ceph mon services will run
+ceph-mon_hosts:
+  controller00:
+    ip: 172.29.236.11
+  controller01:
+    ip: 172.29.236.12
+  controller02:
+    ip: 172.29.236.13
+
+# The nodes that the Ceph OSD disks will be running on
+ceph-osd_hosts:
+  compute00:
+    ip: 172.29.236.14
+  compute01:
+    ip: 172.29.236.15
diff --git a/xci/file/ha/user_ceph.yml b/xci/file/ha/user_ceph.yml
new file mode 100644
index 00000000..9d5f13a9
--- /dev/null
+++ b/xci/file/ha/user_ceph.yml
@@ -0,0 +1,16 @@
+---
+# The OSA ceph_client role does not support loading IPs from an inventory group,
+# so we have to feed it a list of IPs
+# yamllint disable rule:line-length
+ceph_mons: "[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]"
+# yamllint enable rule:line-length
+cinder_backends:
+  "RBD":
+    volume_driver: cinder.volume.drivers.rbd.RBDDriver
+    rbd_pool: volumes
+    rbd_ceph_conf: /etc/ceph/ceph.conf
+    rbd_store_chunk_size: 8
+    volume_backend_name: rbddriver
+    rbd_user: cinder
+    rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"
+    report_discard_supported: true
diff --git a/xci/file/ha/user_variables_ceph.yml b/xci/file/ha/user_variables_ceph.yml
new file mode 100644
index 00000000..8f708990
--- /dev/null
+++ b/xci/file/ha/user_variables_ceph.yml
@@ -0,0 +1,32 @@
+---
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## ceph-ansible settings
+devices: [/dev/loop0, /dev/loop1, /dev/loop2]
+common_single_host_mode: true
+monitor_interface: eth1
+public_network: "172.29.236.0/22"
+cluster_network: "172.29.244.0/22"
+journal_size: 100
+journal_collocation: true
+pool_default_pg_num: 32
+openstack_config: true # Ceph ansible automatically creates pools & keys
+cinder_ceph_client: cinder
+cinder_default_volume_type: RBD
+glance_ceph_client: glance
+glance_default_store: rbd
+glance_rbd_store_pool: images
+nova_libvirt_images_rbd_pool: vms
+nfs_file_gw: False
diff --git a/xci/file/mini/ceph.yml b/xci/file/mini/ceph.yml
new file mode 100644
index 00000000..5c09b471
--- /dev/null
+++ b/xci/file/mini/ceph.yml
@@ -0,0 +1,9 @@
+# The infra nodes where the Ceph mon services will run
+ceph-mon_hosts:
+  controller00:
+    ip: 172.29.236.11
+
+# The nodes that the Ceph OSD disks will be running on
+ceph-osd_hosts:
+  compute00:
+    ip: 172.29.236.12
diff --git a/xci/file/mini/user_ceph.yml b/xci/file/mini/user_ceph.yml
new file mode 100644
index 00000000..9d5f13a9
--- /dev/null
+++ b/xci/file/mini/user_ceph.yml
@@ -0,0 +1,16 @@
+---
+# The OSA ceph_client role does not support loading IPs from an inventory group,
+# so we have to feed it a list of IPs
+# yamllint disable rule:line-length
+ceph_mons: "[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]"
+# yamllint enable rule:line-length
+cinder_backends:
+  "RBD":
+    volume_driver: cinder.volume.drivers.rbd.RBDDriver
+    rbd_pool: volumes
+    rbd_ceph_conf: /etc/ceph/ceph.conf
+    rbd_store_chunk_size: 8
+    volume_backend_name: rbddriver
+    rbd_user: cinder
+    rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"
+    report_discard_supported: true
diff --git a/xci/file/mini/user_variables_ceph.yml b/xci/file/mini/user_variables_ceph.yml
new file mode 100644
index 00000000..8f708990
--- /dev/null
+++ b/xci/file/mini/user_variables_ceph.yml
@@ -0,0 +1,32 @@
+---
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## ceph-ansible settings
+devices: [/dev/loop0, /dev/loop1, /dev/loop2]
+common_single_host_mode: true
+monitor_interface: eth1
+public_network: "172.29.236.0/22"
+cluster_network: "172.29.244.0/22"
+journal_size: 100
+journal_collocation: true
+pool_default_pg_num: 32
+openstack_config: true # Ceph ansible automatically creates pools & keys
+cinder_ceph_client: cinder
+cinder_default_volume_type: RBD
+glance_ceph_client: glance
+glance_default_store: rbd
+glance_rbd_store_pool: images
+nova_libvirt_images_rbd_pool: vms
+nfs_file_gw: False
diff --git a/xci/file/noha/ceph.yml b/xci/file/noha/ceph.yml
new file mode 100644
index 00000000..0deb522e
--- /dev/null
+++ b/xci/file/noha/ceph.yml
@@ -0,0 +1,11 @@
+# The infra nodes where the Ceph mon services will run
+ceph-mon_hosts:
+  controller00:
+    ip: 172.29.236.11
+
+# The nodes that the Ceph OSD disks will be running on
+ceph-osd_hosts:
+  compute00:
+    ip: 172.29.236.12
+  compute01:
+    ip: 172.29.236.13
diff --git a/xci/file/noha/user_ceph.yml b/xci/file/noha/user_ceph.yml
new file mode 100644
index 00000000..9d5f13a9
--- /dev/null
+++ b/xci/file/noha/user_ceph.yml
@@ -0,0 +1,16 @@
+---
+# The OSA ceph_client role does not support loading IPs from an inventory group,
+# so we have to feed it a list of IPs
+# yamllint disable rule:line-length
+ceph_mons: "[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]"
+# yamllint enable rule:line-length
+cinder_backends:
+  "RBD":
+    volume_driver: cinder.volume.drivers.rbd.RBDDriver
+    rbd_pool: volumes
+    rbd_ceph_conf: /etc/ceph/ceph.conf
+    rbd_store_chunk_size: 8
+    volume_backend_name: rbddriver
+    rbd_user: cinder
+    rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"
+    report_discard_supported: true
diff --git a/xci/file/noha/user_variables_ceph.yml b/xci/file/noha/user_variables_ceph.yml
new file mode 100644
index 00000000..8f708990
--- /dev/null
+++ b/xci/file/noha/user_variables_ceph.yml
@@ -0,0 +1,32 @@
+---
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## ceph-ansible settings
+devices: [/dev/loop0, /dev/loop1, /dev/loop2]
+common_single_host_mode: true
+monitor_interface: eth1
+public_network: "172.29.236.0/22"
+cluster_network: "172.29.244.0/22"
+journal_size: 100
+journal_collocation: true
+pool_default_pg_num: 32
+openstack_config: true # Ceph ansible automatically creates pools & keys
+cinder_ceph_client: cinder
+cinder_default_volume_type: RBD
+glance_ceph_client: glance
+glance_default_store: rbd
+glance_rbd_store_pool: images
+nova_libvirt_images_rbd_pool: vms
+nfs_file_gw: False
diff --git a/xci/playbooks/configure-opnfvhost.yml b/xci/playbooks/configure-opnfvhost.yml
index faae623f..9b12584f 100644
--- a/xci/playbooks/configure-opnfvhost.yml
+++ b/xci/playbooks/configure-opnfvhost.yml
@@ -79,6 +79,14 @@
       shell: "/bin/cp -rf {{XCI_FLAVOR_ANSIBLE_FILE_PATH}}/user_*.yml {{OPENSTACK_OSA_ETC_PATH}}"
     - name: copy cinder.yml
       shell: "/bin/cp -rf {{OPNFV_RELENG_PATH}}/xci/file/cinder.yml {{OPENSTACK_OSA_ETC_PATH}}/env.d"
+    - block:
+        - name: copy ceph.yml
+          shell: "/bin/cp -rf {{XCI_FLAVOR_ANSIBLE_FILE_PATH}}/ceph.yml {{OPENSTACK_OSA_ETC_PATH}}/conf.d/"
+        - name: copy user_ceph.yml
+          shell: "/bin/cp -rf {{XCI_FLAVOR_ANSIBLE_FILE_PATH}}/user_ceph.yml {{OPENSTACK_OSA_ETC_PATH}}/user_ceph.yml"
+        - name: copy user_variables_ceph.yml
+          shell: "/bin/cp -rf {{XCI_FLAVOR_ANSIBLE_FILE_PATH}}/user_variables_ceph.yml {{OPENSTACK_OSA_ETC_PATH}}/user_variables_ceph.yml"
+      when: XCI_CEPH_ENABLED == "true"
     # TODO: We need to get rid of this as soon as the issue is fixed upstream
     - name: change the haproxy state from disable to enable
       replace:
diff --git a/xci/playbooks/configure-targethosts.yml b/xci/playbooks/configure-targethosts.yml
index 4cee1df8..b7b09cfb 100644
--- a/xci/playbooks/configure-targethosts.yml
+++ b/xci/playbooks/configure-targethosts.yml
@@ -24,6 +24,7 @@
   remote_user: root
   vars_files:
     - ../var/flavor-vars.yml
+    - ../var/opnfv.yml
   pre_tasks:
     - name: Load distribution variables
       include_vars:
@@ -32,6 +33,8 @@
     - role: configure-network
     # we need to force sync time with ntp or the nodes will be out of sync timewise
     - role: synchronize-time
+    - role: configure-ceph
+      when: XCI_CEPH_ENABLED == "true"
 
 - hosts: compute00
   remote_user: root
diff --git a/xci/playbooks/roles/configure-ceph/tasks/main.yml b/xci/playbooks/roles/configure-ceph/tasks/main.yml
new file mode 100644
index 00000000..93722601
--- /dev/null
+++ b/xci/playbooks/roles/configure-ceph/tasks/main.yml
@@ -0,0 +1,47 @@
+---
+# Copyright 2016, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- name: Create sparse ceph OSD files
+  command: truncate -s {{ host_loopback_ceph_size }} /var/{{ item }}.img
+  args:
+    creates: "/var/{{ item }}.img"
+  with_items: "{{ ceph_osd_images }}"
+  register: ceph_create
+  changed_when: false
+
+- name: Create the ceph loopback device
+  command: losetup -f /var/{{ item.item }}.img --show
+  with_items: "{{ ceph_create.results }}"
+  register: ceph_loopback
+  when: not item|skipped
+  changed_when: false
+
+- debug:
+    msg: "{{ ceph_loopback.results | map(attribute='stdout') | list | to_yaml | trim }}"
+- name: register ceph_loopback to localhost
+  set_fact:
+    fact_for_ceph: "{{ ceph_loopback.results | map(attribute='stdout') | list | to_yaml | trim }}"
+
+- name: Ensure that rc.local exists
+  file:
+    path: "{{ rc_local }}"
+    state: touch
+    mode: "u+x"
+
+- name: Create ceph loopback at boot time
+  lineinfile:
+    dest: "{{ rc_local }}"
+    line: "losetup -f /var/{{ item }}.img"
+    insertbefore: "{{ rc_local_insert_before }}"
+  with_items: "{{ ceph_osd_images }}"
diff --git a/xci/playbooks/roles/configure-ceph/vars/main.yml b/xci/playbooks/roles/configure-ceph/vars/main.yml
new file mode 100644
index 00000000..28e2ad06
--- /dev/null
+++ b/xci/playbooks/roles/configure-ceph/vars/main.yml
@@ -0,0 +1,18 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+host_loopback_ceph_size: "100G"
+rc_local: /etc/rc.local
+rc_local_insert_before: "^exit 0$"
+bootstrap_host_data_disk_device: null
+bootstrap_host_data_disk_device_force: "no"
+ceph_osd_images:
+  - 'ceph1'
+  - 'ceph2'
+  - 'ceph3'
diff --git a/xci/var/opnfv.yml b/xci/var/opnfv.yml
index b1a897f9..7dd9b46c 100644
--- a/xci/var/opnfv.yml
+++ b/xci/var/opnfv.yml
@@ -32,6 +32,7 @@ OPNFV_HOST_IP: "{{ lookup('env','OPNFV_HOST_IP') }}"
 OPNFV_SSH_HOST_KEYS_PATH: "{{ lookup('env', 'OPNFV_SSH_HOST_KEYS_PATH') }}"
 XCI_EXTRA_VARS_PATH: "{{ lookup('env', 'XCI_EXTRA_VARS_PATH') }}"
 XCI_SSL_SUBJECT: "{{ lookup('env', 'XCI_SSL_SUBJECT') }}"
+XCI_CEPH_ENABLED: "{{ lookup('env', 'XCI_CEPH_ENABLED') }}"
 
 # install docker on opnfv host only if we are running as part of CI
 opnfv_required_packages: