Merge "add ceph integration with compass"

5 files changed, 208 insertions, 0 deletions

diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml
new file mode 100644
index 00000000..f897c944
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml
@@ -0,0 +1,49 @@
+---
+- name: create ceph cluster
+  shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy new {{ ceph_cluster_hosts.stdout_lines[0] }}
+  tags:
+    - create_ceph_cluster
+
+- name: install ceph for every nodes includes jumpserver
+  shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy install {{ ceph_cluster_hosts.stdout_lines[0]}}
+
+- name: create monitor node in controller group
+  shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy --overwrite-conf mon create-initial
+
+- name: copy create_osd.sh to host1
+  copy: src=create_osd.sh dest=~/create_osd.sh mode=0777
+  tags:
+    - create_osd
+
+- name: copy create_osd.sh to other nodes
+  shell: scp -o StrictHostKeyChecking=no ~/create_osd.sh {{ item }}:~/
+  with_items:
+    - "{{ groups['controller'] }}"
+  tags:
+    - create_osd
+
+- name: create osd
+  shell: ssh -o StrictHostKeyChecking=no -t {{ item }} "~/create_osd.sh"
+  with_items:
+    - "{{ groups['controller'] }}"
+  tags:
+    - create_osd
+
+- name: prepare create osd
+  shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy osd prepare {{ item }}:/var/local/osd
+  with_items:
+    - "{{ groups['controller'] }}"
+  tags:
+    - create_osd
+
+
+- name: activate osd
+  shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy osd activate {{ item }}:/var/local/osd && ceph-deploy osd activate {{ item }}:/var/local/osd
+  with_items:
+    - "{{ groups['controller'] }}"
+  tags:
+    - create_osd
+    - activate_osd
+
+- name: create admin node
+  shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy admin {{ ceph_cluster_hosts.stdout_lines[0] }}
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml
new file mode 100644
index 00000000..34ce6fdc
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml
@@ -0,0 +1,22 @@
+---
+
+- name: modify glance-api.conf for ceph
+  shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(default_store\).*/\1 = rbd/' /etc/glance/glance-api.conf && sed -i '/^\[glance_store/a stores = rbd\nrbd_store_pool = images\nrbd_store_user = glance\nrbd_store_ceph_conf = /etc/ceph/ceph.conf\nrbd_store_chunk_size = 8' /etc/glance/glance-api.conf && sudo glance-control api restart"
+  with_items:
+    - "{{ groups['controller'] }}"
+  tags:
+    - ceph_conf_glance
+
+- name: modify cinder.conf for ceph
+  shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}' /etc/cinder/cinder.conf && sudo service cinder-volume restart"
+  with_items:
+    - "{{ groups['compute'] }}"
+  tags:
+    - ceph_conf_cinder
+
+- name: modify nova.conf for ceph
+  shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(images_type\).*/\1 = rbd/' /etc/nova/nova-compute.conf && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && sudo service nova-compute restart"
+  with_items:
+    - "{{ groups['compute'] }}"
+  tags:
+    - ceph_conf_nova
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml
new file mode 100644
index 00000000..41c24709
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml
@@ -0,0 +1,59 @@
+---
+
+- name: create pool
+  shell: ceph osd pool create {{ item }} 128
+  with_items:
+    - volumes
+    - images
+    - backups
+    - vms
+
+- name: create ceph users for openstack
+  shell: ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images' && ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
+
+- name: send glance key to controller nodes
+  shell: ceph auth get-or-create client.glance | ssh {{ item }} sudo tee /etc/ceph/ceph.client.glance.keyring && ssh {{ item }} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
+  with_items:
+    - "{{ groups['controller'] }}"
+
+- name: send cinder key to compute nodes
+  shell: ceph auth get-or-create client.cinder | ssh {{ item }} sudo tee /etc/ceph/ceph.client.cinder.keyring && ssh {{ item }} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
+  with_items:
+    - "{{ groups['compute'] }}"
+  tags:
+    - ceph_send_key
+
+- name: copy cinder key to compute node
+  shell: ceph auth get-key client.cinder | ssh {{ item }} tee client.cinder.key
+  with_items:
+    - "{{ groups['compute'] }}"
+  tags:
+    - ceph_copy_secret
+
+- name: gen ceph uuid
+  shell: uuidgen
+  register: ceph_uuid
+  tags:
+    - ceph_copy_secret
+
+- name: gen template secret.xml
+  template: src=secret.j2 dest=~/secret.xml mode=0777
+  tags:
+    - ceph_copy_secret
+
+- name: copy secret.xml to compute nodes
+  shell: scp -o StrictHostKeyChecking=no ~/secret.xml {{ item }}:~/secret.xml
+  with_items:
+    - "{{ groups['compute'] }}"
+  tags:
+    - ceph_copy_secret
+
+- name: create key for libvirt on compute nodes
+  shell: ssh -o StrictHostKeyChecking=no -t {{ item }} "virsh secret-define --file secret.xml && virsh secret-set-value --secret {{ ceph_uuid.stdout_lines[0] }} --base64 \$(cat client.cinder.key) && rm client.cinder.key secret.xml"
+  with_items:
+    - "{{ groups['compute'] }}"
+  tags:
+    - ceph_copy_secret
+
+
+
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml
new file mode 100644
index 00000000..cacc48d8
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml
@@ -0,0 +1,51 @@
+---
+- include_vars: "{{ ansible_os_family }}.yml"
+
+- name: copy rsa secret key from compass-core to host1
+  copy: src=/root/.ssh/id_rsa dest=/root/.ssh/id_rsa mode=0600
+
+- name: copy rsa pub key from compass-core to host1
+  copy: src=/root/.ssh/id_rsa.pub dest=/root/.ssh/id_rsa.pub mode=0644
+
+- name: gen ssh key for no-ask access to every nodes
+  shell: if [ ! -f ~/.ssh/id_rsa.pub ]; then ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N ""; else echo "already gen ssh key!"; fi;
+
+- name: copy key to node
+  shell: ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no root@{{ item }}
+  with_items:
+    - "{{ groups['all'] }}"
+
+- name: install ceph-related packages
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  with_items: packages | union(packages_noarch)
+
+- name: clear cluster_hosts_name
+  file: path=cluster_hosts_name state=absent
+  tags:
+    - create_ceph_cluster
+    - ceph_install
+
+- name: echo hosts name to file
+  shell: echo -n "{{ item }} " >> cluster_hosts_name
+  with_items:
+    - "{{ groups['all'] }}"
+  tags:
+    - create_ceph_cluster
+    - ceph_install
+
+- name: register hosts name
+  shell: cat cluster_hosts_name
+  register: ceph_cluster_hosts
+  tags:
+    - create_ceph_cluster
+    - echo_hosts
+    - ceph_install
+
+- name: gen clean ceph script
+  template: src=clean_ceph.j2 dest=~/clean_ceph.sh mode=0777
+
+- name: clean ceph data of last deploy
+  shell: ~/clean_ceph.sh
+
+- name: mkdir for ceph-cluster
+  file: path={{ ceph_cluster_dir[0] }} state=directory mode=0755
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml
new file mode 100644
index 00000000..a1a9127d
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml
@@ -0,0 +1,27 @@
+---
+- include: ceph_setup_env.yml
+  when: inventory_hostname == "{{ groups['controller'][0] }}"
+  tags:
+    - ceph_deploy
+    - ceph_setup_env
+    - ceph_install
+
+- include: ceph_install.yml
+  when: inventory_hostname == "{{ groups['controller'][0] }}"
+  tags:
+    - ceph_deploy
+    - ceph_install
+
+- include: ceph_openstack_pre.yml
+  when: inventory_hostname == "{{ groups['controller'][0] }}"
+  tags:
+    - ceph_deploy
+    - ceph_openstack_pre
+    - ceph_openstack
+
+- include: ceph_openstack_conf.yml
+  when: inventory_hostname == "{{ groups['controller'][0] }}"
+  tags:
+    - ceph_deploy
+    - ceph_openstack_conf
+    - ceph_openstack