From 2616357bbf1d5cfc1f7e3923e12b8adfc5627fc6 Mon Sep 17 00:00:00 2001 From: grakiss Date: Wed, 13 Jan 2016 14:59:11 +0800 Subject: deploy CEPH nodes parallelly JIRA:COMPASS-264 - do not use ceph-deploy - do not need centeral node to deploy ceph Change-Id: I0f9c459060a2fccbad73a323697ec23ea2115410 Signed-off-by: grakiss --- .../ansible/roles/ceph-config/files/create_osd.sh | 31 ++++++++++++ .../roles/ceph-config/tasks/create_config.yml | 59 ++++++++++++++++++++++ .../ansible/roles/ceph-config/tasks/main.yml | 5 ++ .../ansible/roles/ceph-config/templates/ceph.j2 | 25 +++++++++ .../roles/ceph-config/templates/create_monmap.j2 | 5 ++ .../roles/ceph-config/templates/dump_var.j2 | 8 +++ 6 files changed, 133 insertions(+) create mode 100755 deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh create mode 100755 deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml create mode 100755 deploy/adapters/ansible/roles/ceph-config/tasks/main.yml create mode 100755 deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2 create mode 100644 deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2 create mode 100755 deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2 (limited to 'deploy/adapters/ansible/roles/ceph-config') diff --git a/deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh b/deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh new file mode 100755 index 00000000..2c9e57f1 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-config/files/create_osd.sh @@ -0,0 +1,31 @@ +if [ -d "/var/local/osd" ]; then +echo "clear /var/local/osd" +rm -r /var/local/osd/ +umount /var/local/osd +rm -r /var/local/osd +fi + + +#safe check +ps -ef |grep lvremove |awk '{print $2}' |xargs kill -9 +ps -ef |grep vgremove |awk '{print $2}' |xargs kill -9 +ps -ef |grep vgcreate |awk '{print $2}' |xargs kill -9 +ps -ef |grep lvcreate |awk '{print $2}' |xargs kill -9 + +if [ -L "/dev/storage-volumes/ceph0" ]; then +echo "remove lv vg" +lvremove -f /dev/storage-volumes/ceph0 +fi + + +echo "lvcreate" +lvcreate -l 100%FREE -nceph0 storage-volumes +echo "mkfs" +mkfs.xfs -f /dev/storage-volumes/ceph0 + +if [ ! -d "/var/local/osd" ]; then +echo "mount osd" +mkdir -p /var/local/osd +mount /dev/storage-volumes/ceph0 /var/local/osd +fi + diff --git a/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml b/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml new file mode 100755 index 00000000..0822239e --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-config/tasks/create_config.yml @@ -0,0 +1,59 @@ +- name: gen ceph fsid + shell: uuidgen + register: ceph_fsid + run_once: true + +- name: gen ceph conf + local_action: + module: "template" + src: "ceph.j2" + dest: "/tmp/ceph.conf" + run_once: true + +- name: "make directory for ceph config file" + file: path="/etc/ceph" state="directory" + +- name: copy ceph conf to dest mon node + copy: src="/tmp/ceph.conf" dest="/etc/ceph/ceph.conf" + +- name: install ceph-related packages + action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" + with_items: + - ceph + +- name: gen create monmap script + local_action: template src="create_monmap.j2" dest="/tmp/create_monmap.sh" mode=0755 + run_once: true + +- name: create monmap + script: /tmp/create_monmap.sh + when: inventory_hostname in groups['ceph_mon'] + +- name: create mon.keyring + shell: "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'" + when: inventory_hostname == groups['ceph_mon'][0] + +- name: create admin.keyring + shell: "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'" + when: inventory_hostname == groups['ceph_mon'][0] + +- name: Add the client.admin key to the ceph.mon.keyring + shell: "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring" + when: inventory_hostname == groups['ceph_mon'][0] + +- name: fetch mon.keyring to local + fetch: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring" flat=yes + when: inventory_hostname == groups['ceph_mon'][0] + +- name: fetch client.admin.keyring to local + fetch: src="/etc/ceph/ceph.client.admin.keyring" dest="/tmp/ceph.client.admin.keyring" flat=yes + when: inventory_hostname == groups['ceph_mon'][0] + +- name: copy mon.keyring to remote nodes + copy: src="/tmp/ceph.mon.keyring" dest="/tmp/ceph.mon.keyring" + +- name: copy admin.keyring to remote nodes + copy: src="/tmp/ceph.client.admin.keyring" dest="/etc/ceph/ceph.client.admin.keyring" + + +- meta: flush_handlers diff --git a/deploy/adapters/ansible/roles/ceph-config/tasks/main.yml b/deploy/adapters/ansible/roles/ceph-config/tasks/main.yml new file mode 100755 index 00000000..3512dfb8 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-config/tasks/main.yml @@ -0,0 +1,5 @@ +- include: create_config.yml + tags: + - ceph_config + - ceph_deploy + - ceph_mon diff --git a/deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2 b/deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2 new file mode 100755 index 00000000..bd0e3f5d --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-config/templates/ceph.j2 @@ -0,0 +1,25 @@ +[global] +fsid = {{ ceph_fsid.stdout }} +mon initial members = {{ groups["ceph_mon"] | join(", ")}} +mon host = +{%- for host in groups["ceph_mon"] -%} +{{ ', ' if not loop.first else ''}}{{ ip_settings[host].mgmt.ip }} +{%- endfor %} + +public network = {{ mgmt_cidr }} +cluster network = {{ storage_cidr }} + +auth cluster required = cephx +auth service required = cephx +auth client required = cephx + +osd journal size = 1024 +filestore xattr use omap = true +osd pool default size = 1 +osd pool default min size = 1 +osd pool default pg num = 333 +osd pool default pgp num = 333 +osd crush chooseleaf type = 1 + +debug mon = 1 +debug ms = 0 diff --git a/deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2 b/deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2 new file mode 100644 index 00000000..7d1eb9d7 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-config/templates/create_monmap.j2 @@ -0,0 +1,5 @@ +monmaptool --create --clobber --fsid {{ ceph_fsid.stdout }} +{%- for host in groups['ceph_mon']%} + --add {{host}} {{ ip_settings[host].mgmt.ip }}:6789 +{%- endfor %} + /tmp/monmap diff --git a/deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2 b/deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2 new file mode 100755 index 00000000..a4a9b155 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-config/templates/dump_var.j2 @@ -0,0 +1,8 @@ +HOSTVARS (ANSIBLE GATHERED, group_vars, host_vars) : + +{{ hostvars[inventory_hostname] | to_yaml }} + +PLAYBOOK VARS: + +{{ vars | to_yaml }} + -- cgit 1.2.3-korg