Support compute node expansion with ceph-osd

JIRA: COMPASS-481

- Use public_vip to fetch ceph related conf from controller nodes.
- Add ceph-uuid for libvirt store and fetch for expansion.

Change-Id: I9e93183ac108bd959a1226546fbb89706e3004c9
Signed-off-by: QiLiang <liangqi1@huawei.com>

3 files changed, 33 insertions, 5 deletions

diff --git a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml
index d7c414ee..0496ba97 100755
--- a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml
+++ b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_conf.yml
@@ -28,13 +28,13 @@
   ignore_errors: True
 
 - name: modify cinder.conf for ceph
-  shell: sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/g' /etc/cinder/cinder.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid.stdout_lines[0] }}/g' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}' /etc/cinder/cinder.conf && service {{ cinder_service }} restart
+  shell: sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/g' /etc/cinder/cinder.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid }}/g' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid }}' /etc/cinder/cinder.conf && service {{ cinder_service }} restart
   when: inventory_hostname in groups['compute']
   tags:
     - ceph_conf_cinder
 
 - name: modify nova.conf for ceph
-  shell: sed -i 's/^\(images_type\).*/\1 = rbd/g' /etc/nova/nova-compute.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid.stdout_lines[0] }}/g' /etc/nova/nova-compute.conf  && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && service  {{ nova_service }} restart
+  shell: sed -i 's/^\(images_type\).*/\1 = rbd/g' /etc/nova/nova-compute.conf && sed -i 's/^\(rbd_secret_uuid\).*/\1 = {{ ceph_uuid }}/g' /etc/nova/nova-compute.conf  && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && service  {{ nova_service }} restart
   when: inventory_hostname in groups['compute']
   tags:
     - ceph_conf_nova
diff --git a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml
index 78b71ec0..ece4154f 100755
--- a/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml
+++ b/deploy/adapters/ansible/roles/ceph-openstack/tasks/ceph_openstack_pre.yml
@@ -9,11 +9,39 @@
 ---
 - name: gen ceph uuid
   shell: uuidgen
-  register: ceph_uuid
+  register: ceph_uuid_out
   run_once: true
   tags:
     - ceph_copy_secret
 
+- name: set ceph uuid var
+  set_fact:
+    ceph_uuid: "{{ ceph_uuid_out.stdout_lines[0] }}"
+  when: not compute_expansion
+  tags:
+    - ceph_copy_secret
+
+- name: save ceph uuid for expansion
+  copy: content="{{ ceph_uuid }}" dest=/etc/ceph/ceph.uuid.libvirt
+  when: not compute_expansion and inventory_hostname in groups['controller']
+  tags:
+    - ceph_copy_secret
+
+- name: fetch ceph uuid from controller node for expansion
+  shell: cat /etc/ceph/ceph.uuid.libvirt
+  register: ceph_uuid_out
+  delegate_to: "{{ public_vip.ip }}"
+  when: compute_expansion and inventory_hostname in groups['compute']
+  tags:
+    - ceph_copy_secret
+
+- name: set ceph uuid var for expansion
+  set_fact:
+    ceph_uuid: "{{ ceph_uuid_out.stdout_lines[0] }}"
+  when: compute_expansion and inventory_hostname in groups['compute']
+  tags:
+    - ceph_copy_secret
+
 - name: gen template secret.xml
   local_action:
     module: "template"
@@ -68,7 +96,7 @@
 
 
 - name: create key for libvirt on compute nodes
-  shell: "virsh secret-define --file ~/secret.xml && virsh secret-set-value --secret {{ ceph_uuid.stdout_lines[0] }} --base64 $(cat client.cinder.key)"
+  shell: "virsh secret-define --file ~/secret.xml && virsh secret-set-value --secret {{ ceph_uuid }} --base64 $(cat client.cinder.key)"
   when: inventory_hostname in groups['compute']
   tags:
     - ceph_copy_secret
diff --git a/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2 b/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2
index a0ffc6e3..0195e099 100644
--- a/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2
+++ b/deploy/adapters/ansible/roles/ceph-openstack/templates/secret.j2
@@ -1,5 +1,5 @@
 <secret ephemeral='no' private='no'>
-  <uuid>{{ ceph_uuid.stdout_lines[0] }}</uuid>
+  <uuid>{{ ceph_uuid }}</uuid>
   <usage type='ceph'>
     <name>client.cinder secret</name>
   </usage>