Merge "deploy CEPH nodes parallelly"

6 files changed, 133 insertions, 0 deletions

diff --git a/deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh b/deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh
new file mode 100755
index 00000000..2c9e57f1
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh
@@ -0,0 +1,31 @@
+if [ -d "/var/local/osd" ]; then
+echo "clear /var/local/osd"
+rm -r /var/local/osd/
+umount /var/local/osd
+rm -r /var/local/osd
+fi
+
+
+#safe check
+ps -ef |grep lvremove |awk '{print $2}' |xargs kill -9
+ps -ef |grep vgremove |awk '{print $2}' |xargs kill -9
+ps -ef |grep vgcreate |awk '{print $2}' |xargs kill -9
+ps -ef |grep lvcreate |awk '{print $2}' |xargs kill -9
+
+if [ -L "/dev/storage-volumes/ceph0" ]; then
+echo "remove lv vg"
+lvremove -f /dev/storage-volumes/ceph0
+fi
+
+
+echo "lvcreate"
+lvcreate -l 100%FREE -nceph0 storage-volumes
+echo "mkfs"
+mkfs.xfs -f /dev/storage-volumes/ceph0
+
+if [ ! -d "/var/local/osd" ]; then
+echo "mount osd"
+mkdir -p /var/local/osd
+mount /dev/storage-volumes/ceph0 /var/local/osd
+fi
+
diff --git a/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml b/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml
new file mode 100755
index 00000000..0822239e
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml
@@ -0,0 +1,59 @@
+- name: gen ceph fsid
+  shell: uuidgen
+  register: ceph_fsid
+  run_once: true
+
+- name: gen ceph conf
+  local_action:
+    module: "template"
+    src: "ceph.j2"
+    dest: "/tmp/ceph.conf"
+  run_once: true
+
+- name: "make directory for ceph config file"
+  file: path="/etc/ceph" state="directory"
+
+- name: copy ceph conf to dest mon node
+  copy: src="/tmp/ceph.conf" dest="/etc/ceph/ceph.conf"
+
+- name: install ceph-related packages
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  with_items:
+    - ceph
+
+- name: gen create monmap script
+  local_action: template src="create_monmap.j2" dest="/tmp/create_monmap.sh" mode=0755
+  run_once: true
+
+- name: create monmap
+  script: /tmp/create_monmap.sh
+  when: inventory_hostname in groups['ceph_mon']
+
+- name: create mon.keyring
+  shell: "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'"
+  when: inventory_hostname == groups['ceph_mon'][0]
+
+- name: create admin.keyring
+  shell: "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'"
+  when: inventory_hostname == groups['ceph_mon'][0]
+ 
+- name: Add the client.admin key to the ceph.mon.keyring
+  shell: "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring"
+  when: inventory_hostname == groups['ceph_mon'][0]
+
+- name: fetch mon.keyring to local
+  fetch: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring" flat=yes
+  when: inventory_hostname == groups['ceph_mon'][0]
+
+- name: fetch client.admin.keyring to local
+  fetch: src="/etc/ceph/ceph.client.admin.keyring" dest="/tmp/ceph.client.admin.keyring" flat=yes
+  when: inventory_hostname == groups['ceph_mon'][0]
+
+- name: copy mon.keyring to remote nodes
+  copy: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring"
+
+- name: copy admin.keyring to remote nodes
+  copy: src="/tmp/ceph.client.admin.keyring" dest="/etc/ceph/ceph.client.admin.keyring"
+
+
+- meta: flush_handlers
diff --git a/deploy/adapters/ansible/roles/ceph-config/tasks/main.yml b/deploy/adapters/ansible/roles/ceph-config/tasks/main.yml
new file mode 100755
index 00000000..3512dfb8
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-config/tasks/main.yml
@@ -0,0 +1,5 @@
+- include: create_config.yml
+  tags:
+    - ceph_config
+    - ceph_deploy
+    - ceph_mon
diff --git a/deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2 b/deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2
new file mode 100755
index 00000000..bd0e3f5d
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2
@@ -0,0 +1,25 @@
+[global]
+fsid = {{ ceph_fsid.stdout }}
+mon initial members = {{ groups["ceph_mon"] | join(", ")}}
+mon host = 
+{%- for host in groups["ceph_mon"] -%}
+{{ ', ' if not loop.first else ''}}{{ ip_settings[host].mgmt.ip }}
+{%- endfor %}
+
+public network = {{ mgmt_cidr }}
+cluster network = {{ storage_cidr }}
+
+auth cluster required = cephx
+auth service required = cephx
+auth client required = cephx
+
+osd journal size = 1024
+filestore xattr use omap = true
+osd pool default size = 1 
+osd pool default min size = 1
+osd pool default pg num = 333
+osd pool default pgp num = 333
+osd crush chooseleaf type = 1
+
+debug mon = 1
+debug ms = 0
diff --git a/deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2 b/deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2
new file mode 100644
index 00000000..7d1eb9d7
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2
@@ -0,0 +1,5 @@
+monmaptool --create --clobber --fsid {{ ceph_fsid.stdout }} 
+{%- for host in groups['ceph_mon']%}
+ --add {{host}} {{ ip_settings[host].mgmt.ip }}:6789 
+{%- endfor %}
+ /tmp/monmap 
diff --git a/deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2 b/deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2
new file mode 100755
index 00000000..a4a9b155
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2
@@ -0,0 +1,8 @@
+HOSTVARS (ANSIBLE GATHERED, group_vars, host_vars) :
+
+{{ hostvars[inventory_hostname] | to_yaml }}
+
+PLAYBOOK VARS:
+
+{{ vars | to_yaml }}
+