Support compute node expansion with ceph-osd

JIRA: COMPASS-481

- Use public_vip to fetch ceph related conf from controller nodes.
- Add ceph-uuid for libvirt store and fetch for expansion.

Change-Id: I9e93183ac108bd959a1226546fbb89706e3004c9
Signed-off-by: QiLiang <liangqi1@huawei.com>

10 files changed, 92 insertions, 5 deletions

diff --git a/deploy/adapters/ansible/openstack_mitaka/roles/ceph-osd/tasks/install_osd.yml b/deploy/adapters/ansible/openstack_mitaka/roles/ceph-osd/tasks/install_osd.yml
index 0e476085..35e84cf8 100644
--- a/deploy/adapters/ansible/openstack_mitaka/roles/ceph-osd/tasks/install_osd.yml
+++ b/deploy/adapters/ansible/openstack_mitaka/roles/ceph-osd/tasks/install_osd.yml
@@ -11,6 +11,11 @@
 - name: create osd lv and mount it on /var/local/osd
   script: create_osd.sh
 
+- name: fetch osd keyring from ceph_adm
+  fetch: src="/var/lib/ceph/bootstrap-osd/ceph.keyring" dest="/tmp/ceph.osd.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: copy osd keyring
   copy: src="/tmp/ceph.osd.keyring" dest="/var/lib/ceph/bootstrap-osd/ceph.keyring"
 
diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/ceph-osd/tasks/install_osd.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/ceph-osd/tasks/install_osd.yml
index 16f261ef..363e5e6d 100644
--- a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/ceph-osd/tasks/install_osd.yml
+++ b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/ceph-osd/tasks/install_osd.yml
@@ -11,6 +11,11 @@
 - name: create osd lv and mount it on /var/local/osd
   script: create_osd.sh
 
+- name: fetch osd keyring from ceph_adm
+  fetch: src="/var/lib/ceph/bootstrap-osd/ceph.keyring" dest="/tmp/ceph.osd.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: copy osd keyring
   copy: src="/tmp/ceph.osd.keyring" dest="/var/lib/ceph/bootstrap-osd/ceph.keyring"
 
diff --git a/deploy/adapters/ansible/openstack_newton_xenial/roles/ceph-osd/tasks/install_osd.yml b/deploy/adapters/ansible/openstack_newton_xenial/roles/ceph-osd/tasks/install_osd.yml
index 16f261ef..363e5e6d 100644
--- a/deploy/adapters/ansible/openstack_newton_xenial/roles/ceph-osd/tasks/install_osd.yml
+++ b/deploy/adapters/ansible/openstack_newton_xenial/roles/ceph-osd/tasks/install_osd.yml
@@ -11,6 +11,11 @@
 - name: create osd lv and mount it on /var/local/osd
   script: create_osd.sh
 
+- name: fetch osd keyring from ceph_adm
+  fetch: src="/var/lib/ceph/bootstrap-osd/ceph.keyring" dest="/tmp/ceph.osd.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: copy osd keyring
   copy: src="/tmp/ceph.osd.keyring" dest="/var/lib/ceph/bootstrap-osd/ceph.keyring"
 
diff --git a/deploy/adapters/ansible/openstack_osp9/roles/ceph-osd/tasks/install_osd.yml b/deploy/adapters/ansible/openstack_osp9/roles/ceph-osd/tasks/install_osd.yml
index 0e476085..35e84cf8 100755
--- a/deploy/adapters/ansible/openstack_osp9/roles/ceph-osd/tasks/install_osd.yml
+++ b/deploy/adapters/ansible/openstack_osp9/roles/ceph-osd/tasks/install_osd.yml
@@ -11,6 +11,11 @@
 - name: create osd lv and mount it on /var/local/osd
   script: create_osd.sh
 
+- name: fetch osd keyring from ceph_adm
+  fetch: src="/var/lib/ceph/bootstrap-osd/ceph.keyring" dest="/tmp/ceph.osd.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: copy osd keyring
   copy: src="/tmp/ceph.osd.keyring" dest="/var/lib/ceph/bootstrap-osd/ceph.keyring"
 
diff --git a/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml b/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml
index 891e23e0..771fb6ab 100755
--- a/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml
+++ b/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml
@@ -18,6 +18,11 @@
     dest: "/tmp/ceph.conf"
   when: inventory_hostname in groups['ceph_adm']
 
+- name: fetch ceph conf from ceph_adm
+  fetch: src="/etc/ceph/ceph.conf" dest="/tmp/ceph.conf" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: "make directory for ceph config file"
   file: path="/etc/ceph" state="directory"
 
@@ -53,10 +58,20 @@
   fetch: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring" flat=yes
   when: inventory_hostname in groups['ceph_adm']
 
+- name: fetch mon.keyring from ceph_adm
+  fetch: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: fetch client.admin.keyring to local
   fetch: src="/etc/ceph/ceph.client.admin.keyring" dest="/tmp/ceph.client.admin.keyring" flat=yes
   when: inventory_hostname in groups['ceph_adm']
 
+- name: fetch mon.keyring from ceph_adm
+  fetch: src="/etc/ceph/ceph.client.admin.keyring" dest="/tmp/ceph.client.admin.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: copy mon.keyring to remote nodes
   copy: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring"
 
diff --git a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml
index d7c414ee..0496ba97 100755
--- a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml
+++ b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml
@@ -28,13 +28,13 @@
   ignore_errors: True
 
 - name: modify cinder.conf for ceph
-  shell: sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/g' /etc/cinder/cinder.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid.stdout_lines[0] }}/g' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}' /etc/cinder/cinder.conf && service {{ cinder_service }} restart
+  shell: sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/g' /etc/cinder/cinder.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid }}/g' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid }}' /etc/cinder/cinder.conf && service {{ cinder_service }} restart
   when: inventory_hostname in groups['compute']
   tags:
     - ceph_conf_cinder
 
 - name: modify nova.conf for ceph
-  shell: sed -i 's/^\(images_type\).*/\1 = rbd/g' /etc/nova/nova-compute.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid.stdout_lines[0] }}/g' /etc/nova/nova-compute.conf  && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && service  {{ nova_service }} restart
+  shell: sed -i 's/^\(images_type\).*/\1 = rbd/g' /etc/nova/nova-compute.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid }}/g' /etc/nova/nova-compute.conf  && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && service  {{ nova_service }} restart
   when: inventory_hostname in groups['compute']
   tags:
     - ceph_conf_nova
diff --git a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml
index 78b71ec0..ece4154f 100755
--- a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml
+++ b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml
@@ -9,11 +9,39 @@
 ---
 - name: gen ceph uuid
   shell: uuidgen
-  register: ceph_uuid
+  register: ceph_uuid_out
   run_once: true
   tags:
     - ceph_copy_secret
 
+- name: set ceph uuid var
+  set_fact:
+    ceph_uuid: "{{ ceph_uuid_out.stdout_lines[0] }}"
+  when: not compute_expansion
+  tags:
+    - ceph_copy_secret
+
+- name: save ceph uuid for expansion
+  copy: content="{{ ceph_uuid }}" dest=/etc/ceph/ceph.uuid.libvirt
+  when: not compute_expansion and inventory_hostname in groups['controller']
+  tags:
+    - ceph_copy_secret
+
+- name: fetch ceph uuid from controller node for expansion
+  shell: cat /etc/ceph/ceph.uuid.libvirt
+  register: ceph_uuid_out
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion and inventory_hostname in groups['compute']
+  tags:
+    - ceph_copy_secret
+
+- name: set ceph uuid var for expansion
+  set_fact:
+    ceph_uuid: "{{ ceph_uuid_out.stdout_lines[0] }}"
+  when: compute_expansion and inventory_hostname in groups['compute']
+  tags:
+    - ceph_copy_secret
+
 - name: gen template secret.xml
   local_action:
     module: "template"
@@ -68,7 +96,7 @@
 
 
 - name: create key for libvirt on compute nodes
-  shell: "virsh secret-define --file ~/secret.xml && virsh secret-set-value --secret {{ ceph_uuid.stdout_lines[0] }} --base64 $(cat client.cinder.key)"
+  shell: "virsh secret-define --file ~/secret.xml && virsh secret-set-value --secret {{ ceph_uuid }} --base64 $(cat client.cinder.key)"
   when: inventory_hostname in groups['compute']
   tags:
     - ceph_copy_secret
diff --git a/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2 b/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2
index a0ffc6e3..0195e099 100644
--- a/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2
+++ b/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2
@@ -1,5 +1,5 @@
 <secret ephemeral='no' private='no'>
-  <uuid>{{ ceph_uuid.stdout_lines[0] }}</uuid>
+  <uuid>{{ ceph_uuid }}</uuid>
   <usage type='ceph'>
     <name>client.cinder secret</name>
   </usage>
diff --git a/deploy/adapters/ansible/roles/ceph-osd/tasks/install_osd.yml b/deploy/adapters/ansible/roles/ceph-osd/tasks/install_osd.yml
index e7e4a242..ff99d68a 100644
--- a/deploy/adapters/ansible/roles/ceph-osd/tasks/install_osd.yml
+++ b/deploy/adapters/ansible/roles/ceph-osd/tasks/install_osd.yml
@@ -11,6 +11,11 @@
 - name: create osd lv and mount it on /var/local/osd
   script: create_osd.sh
 
+- name: fetch osd keyring from ceph_adm
+  fetch: src="/var/lib/ceph/bootstrap-osd/ceph.keyring" dest="/tmp/ceph.osd.keyring" flat=yes
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion
+
 - name: copy osd keyring
   copy: src="/tmp/ceph.osd.keyring" dest="/var/lib/ceph/bootstrap-osd/ceph.keyring"
 
diff --git a/deploy/compass_vm.sh b/deploy/compass_vm.sh
index 61e27051..b07b5081 100755
--- a/deploy/compass_vm.sh
+++ b/deploy/compass_vm.sh
@@ -70,8 +70,27 @@ function _inject_dashboard_conf() {
     done
 }
 
+function _inject_ceph_expansion_conf() {
+    for os in mitaka mitaka_xenial newton_xenial osp9; do
+        CONF_TEMPLATES_DIR=/etc/compass/templates/ansible_installer/openstack_$os/vars
+        if [[ "$EXPANSION" == "true" ]]; then
+            cmd="
+                sed -i '/compute_expansion/d' ${CONF_TEMPLATES_DIR}/HA-ansible-multinodes.tmpl; \
+                echo compute_expansion: True >> ${CONF_TEMPLATES_DIR}/HA-ansible-multinodes.tmpl; \
+            "
+        else
+            cmd="
+                sed -i '/compute_expansion/d' ${CONF_TEMPLATES_DIR}/HA-ansible-multinodes.tmpl; \
+                echo compute_expansion: False >> ${CONF_TEMPLATES_DIR}/HA-ansible-multinodes.tmpl; \
+            "
+        fi
+        exec_cmd_on_compass $cmd
+    done
+}
+
 function inject_compass_conf() {
     _inject_dashboard_conf
+    _inject_ceph_expansion_conf
 }
 
 function refresh_compass_core () {