summaryrefslogtreecommitdiffstats
path: root/deploy
diff options
context:
space:
mode:
Diffstat (limited to 'deploy')
-rw-r--r--deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml1
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh41
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml49
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml22
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml59
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml51
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml27
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo7
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j28
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j26
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml18
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml18
-rw-r--r--deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml5
-rw-r--r--deploy/adapters/ansible/roles/ha/templates/haproxy.cfg2
-rwxr-xr-xdeploy/network.sh74
15 files changed, 369 insertions, 19 deletions
diff --git a/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml
index af0727c7..692650b5 100644
--- a/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml
@@ -53,4 +53,5 @@
remote_user: root
sudo: True
roles:
+ - ceph-deploy
- monitor
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh b/deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh
new file mode 100644
index 00000000..2535f364
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh
@@ -0,0 +1,41 @@
+if [ -d "/var/local/osd" ]; then
+echo "clear /var/local/osd"
+rm -r /var/local/osd/
+umount /var/local/osd
+rm -r /var/local/osd
+fi
+
+if [ ! -d "/ceph/images" ]; then
+mkdir -p /ceph/images
+fi
+
+rm /ceph/images/ceph-volumes.img
+
+if [ ! -f "/ceph/images/ceph-volumes.img" ]; then
+echo "create ceph-volumes.img"
+dd if=/dev/zero of=/ceph/images/ceph-volumes.img bs=1M seek=12288 count=0 oflag=direct
+sgdisk -g --clear /ceph/images/ceph-volumes.img
+fi
+
+if [ -L "/dev/ceph-volumes/ceph0" ]; then
+echo "remove lv vg"
+lvremove /dev/ceph-volumes/ceph0
+vgremove ceph-volumes
+rm -r /dev/ceph-volumes
+fi
+
+losetup -d /dev/loop0
+
+echo "vgcreate"
+vgcreate ceph-volumes $(sudo losetup --show -f /ceph/images/ceph-volumes.img)
+echo "lvcreate"
+sudo lvcreate -L9G -nceph0 ceph-volumes
+echo "mkfs"
+mkfs.xfs -f /dev/ceph-volumes/ceph0
+
+if [ ! -d "/var/local/osd" ]; then
+echo "mount osd"
+mkdir -p /var/local/osd
+mount /dev/ceph-volumes/ceph0 /var/local/osd
+fi
+
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml
new file mode 100644
index 00000000..f897c944
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml
@@ -0,0 +1,49 @@
+---
+- name: create ceph cluster
+ shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy new {{ ceph_cluster_hosts.stdout_lines[0] }}
+ tags:
+ - create_ceph_cluster
+
+- name: install ceph for every nodes includes jumpserver
+ shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy install {{ ceph_cluster_hosts.stdout_lines[0]}}
+
+- name: create monitor node in controller group
+ shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy --overwrite-conf mon create-initial
+
+- name: copy create_osd.sh to host1
+ copy: src=create_osd.sh dest=~/create_osd.sh mode=0777
+ tags:
+ - create_osd
+
+- name: copy create_osd.sh to other nodes
+ shell: scp -o StrictHostKeyChecking=no ~/create_osd.sh {{ item }}:~/
+ with_items:
+ - "{{ groups['controller'] }}"
+ tags:
+ - create_osd
+
+- name: create osd
+ shell: ssh -o StrictHostKeyChecking=no -t {{ item }} "~/create_osd.sh"
+ with_items:
+ - "{{ groups['controller'] }}"
+ tags:
+ - create_osd
+
+- name: prepare create osd
+ shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy osd prepare {{ item }}:/var/local/osd
+ with_items:
+ - "{{ groups['controller'] }}"
+ tags:
+ - create_osd
+
+
+- name: activate osd
+ shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy osd activate {{ item }}:/var/local/osd && ceph-deploy osd activate {{ item }}:/var/local/osd
+ with_items:
+ - "{{ groups['controller'] }}"
+ tags:
+ - create_osd
+ - activate_osd
+
+- name: create admin node
+ shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy admin {{ ceph_cluster_hosts.stdout_lines[0] }}
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml
new file mode 100644
index 00000000..34ce6fdc
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml
@@ -0,0 +1,22 @@
+---
+
+- name: modify glance-api.conf for ceph
+ shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(default_store\).*/\1 = rbd/' /etc/glance/glance-api.conf && sed -i '/^\[glance_store/a stores = rbd\nrbd_store_pool = images\nrbd_store_user = glance\nrbd_store_ceph_conf = /etc/ceph/ceph.conf\nrbd_store_chunk_size = 8' /etc/glance/glance-api.conf && sudo glance-control api restart"
+ with_items:
+ - "{{ groups['controller'] }}"
+ tags:
+ - ceph_conf_glance
+
+- name: modify cinder.conf for ceph
+ shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}' /etc/cinder/cinder.conf && sudo service cinder-volume restart"
+ with_items:
+ - "{{ groups['compute'] }}"
+ tags:
+ - ceph_conf_cinder
+
+- name: modify nova.conf for ceph
+ shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(images_type\).*/\1 = rbd/' /etc/nova/nova-compute.conf && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && sudo service nova-compute restart"
+ with_items:
+ - "{{ groups['compute'] }}"
+ tags:
+ - ceph_conf_nova
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml
new file mode 100644
index 00000000..41c24709
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml
@@ -0,0 +1,59 @@
+---
+
+- name: create pool
+ shell: ceph osd pool create {{ item }} 128
+ with_items:
+ - volumes
+ - images
+ - backups
+ - vms
+
+- name: create ceph users for openstack
+ shell: ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images' && ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
+
+- name: send glance key to controller nodes
+ shell: ceph auth get-or-create client.glance | ssh {{ item }} sudo tee /etc/ceph/ceph.client.glance.keyring && ssh {{ item }} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
+ with_items:
+ - "{{ groups['controller'] }}"
+
+- name: send cinder key to compute nodes
+ shell: ceph auth get-or-create client.cinder | ssh {{ item }} sudo tee /etc/ceph/ceph.client.cinder.keyring && ssh {{ item }} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
+ with_items:
+ - "{{ groups['compute'] }}"
+ tags:
+ - ceph_send_key
+
+- name: copy cinder key to compute node
+ shell: ceph auth get-key client.cinder | ssh {{ item }} tee client.cinder.key
+ with_items:
+ - "{{ groups['compute'] }}"
+ tags:
+ - ceph_copy_secret
+
+- name: gen ceph uuid
+ shell: uuidgen
+ register: ceph_uuid
+ tags:
+ - ceph_copy_secret
+
+- name: gen template secret.xml
+ template: src=secret.j2 dest=~/secret.xml mode=0777
+ tags:
+ - ceph_copy_secret
+
+- name: copy secret.xml to compute nodes
+ shell: scp -o StrictHostKeyChecking=no ~/secret.xml {{ item }}:~/secret.xml
+ with_items:
+ - "{{ groups['compute'] }}"
+ tags:
+ - ceph_copy_secret
+
+- name: create key for libvirt on compute nodes
+ shell: ssh -o StrictHostKeyChecking=no -t {{ item }} "virsh secret-define --file secret.xml && virsh secret-set-value --secret {{ ceph_uuid.stdout_lines[0] }} --base64 \$(cat client.cinder.key) && rm client.cinder.key secret.xml"
+ with_items:
+ - "{{ groups['compute'] }}"
+ tags:
+ - ceph_copy_secret
+
+
+
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml
new file mode 100644
index 00000000..cacc48d8
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml
@@ -0,0 +1,51 @@
+---
+- include_vars: "{{ ansible_os_family }}.yml"
+
+- name: copy rsa secret key from compass-core to host1
+ copy: src=/root/.ssh/id_rsa dest=/root/.ssh/id_rsa mode=0600
+
+- name: copy rsa pub key from compass-core to host1
+ copy: src=/root/.ssh/id_rsa.pub dest=/root/.ssh/id_rsa.pub mode=0644
+
+- name: gen ssh key for no-ask access to every nodes
+ shell: if [ ! -f ~/.ssh/id_rsa.pub ]; then ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N ""; else echo "already gen ssh key!"; fi;
+
+- name: copy key to node
+ shell: ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no root@{{ item }}
+ with_items:
+ - "{{ groups['all'] }}"
+
+- name: install ceph-related packages
+ action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+ with_items: packages | union(packages_noarch)
+
+- name: clear cluster_hosts_name
+ file: path=cluster_hosts_name state=absent
+ tags:
+ - create_ceph_cluster
+ - ceph_install
+
+- name: echo hosts name to file
+ shell: echo -n "{{ item }} " >> cluster_hosts_name
+ with_items:
+ - "{{ groups['all'] }}"
+ tags:
+ - create_ceph_cluster
+ - ceph_install
+
+- name: register hosts name
+ shell: cat cluster_hosts_name
+ register: ceph_cluster_hosts
+ tags:
+ - create_ceph_cluster
+ - echo_hosts
+ - ceph_install
+
+- name: gen clean ceph script
+ template: src=clean_ceph.j2 dest=~/clean_ceph.sh mode=0777
+
+- name: clean ceph data of last deploy
+ shell: ~/clean_ceph.sh
+
+- name: mkdir for ceph-cluster
+ file: path={{ ceph_cluster_dir[0] }} state=directory mode=0755
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml
new file mode 100644
index 00000000..a1a9127d
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml
@@ -0,0 +1,27 @@
+---
+- include: ceph_setup_env.yml
+ when: inventory_hostname == "{{ groups['controller'][0] }}"
+ tags:
+ - ceph_deploy
+ - ceph_setup_env
+ - ceph_install
+
+- include: ceph_install.yml
+ when: inventory_hostname == "{{ groups['controller'][0] }}"
+ tags:
+ - ceph_deploy
+ - ceph_install
+
+- include: ceph_openstack_pre.yml
+ when: inventory_hostname == "{{ groups['controller'][0] }}"
+ tags:
+ - ceph_deploy
+ - ceph_openstack_pre
+ - ceph_openstack
+
+- include: ceph_openstack_conf.yml
+ when: inventory_hostname == "{{ groups['controller'][0] }}"
+ tags:
+ - ceph_deploy
+ - ceph_openstack_conf
+ - ceph_openstack
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo b/deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo
new file mode 100644
index 00000000..e6b7c7d9
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo
@@ -0,0 +1,7 @@
+[ceph-noarch]
+name=Ceph noarch packages
+baseurl=http://ceph.com/rpm-giant/el6/noarch
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j2 b/deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j2
new file mode 100644
index 00000000..a089167d
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j2
@@ -0,0 +1,8 @@
+if [ -d "{{ ceph_cluster_dir[0] }}" ]; then
+cd {{ ceph_cluster_dir[0] }}
+ceph-deploy purge {{ ceph_cluster_hosts.stdout_lines[0] }}
+ceph-deploy purgedata {{ ceph_cluster_hosts.stdout_lines[0] }}
+ceph-deploy forgetkeys
+cd ..
+rm -r {{ ceph_cluster_dir[0] }}
+fi
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j2 b/deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j2
new file mode 100644
index 00000000..a0ffc6e3
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j2
@@ -0,0 +1,6 @@
+<secret ephemeral='no' private='no'>
+ <uuid>{{ ceph_uuid.stdout_lines[0] }}</uuid>
+ <usage type='ceph'>
+ <name>client.cinder secret</name>
+ </usage>
+</secret>
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml b/deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml
new file mode 100644
index 00000000..5ed6cc10
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml
@@ -0,0 +1,18 @@
+---
+packages:
+ - ceph-deploy
+ - python-flask
+ - libgoogle-perftools4
+ - libleveldb1
+ - liblttng-ust0
+ - libsnappy1
+ - librbd1
+ - librados2
+ - python-ceph
+ - ceph
+ - ceph-mds
+ - ceph-common
+ - ceph-fs-common
+ - gdisk
+
+services: []
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml b/deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml
new file mode 100644
index 00000000..5ed6cc10
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml
@@ -0,0 +1,18 @@
+---
+packages:
+ - ceph-deploy
+ - python-flask
+ - libgoogle-perftools4
+ - libleveldb1
+ - liblttng-ust0
+ - libsnappy1
+ - librbd1
+ - librados2
+ - python-ceph
+ - ceph
+ - ceph-mds
+ - ceph-common
+ - ceph-fs-common
+ - gdisk
+
+services: []
diff --git a/deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml b/deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml
new file mode 100644
index 00000000..71ace4a5
--- /dev/null
+++ b/deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml
@@ -0,0 +1,5 @@
+---
+packages_noarch: []
+
+ceph_cluster_dir:
+ - /root/ceph-cluster
diff --git a/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg
index 4b20db03..169182da 100644
--- a/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg
+++ b/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg
@@ -6,7 +6,7 @@ global
group haproxy
maxconn 4000
pidfile /var/run/haproxy/haproxy.pid
- #log 127.0.0.1 local0
+ log 127.0.0.1 local0
tune.bufsize 1000000
stats socket /var/run/haproxy.sock
stats timeout 2m
diff --git a/deploy/network.sh b/deploy/network.sh
index 864ec011..a5344f58 100755
--- a/deploy/network.sh
+++ b/deploy/network.sh
@@ -8,27 +8,61 @@ function destroy_bridge()
{
bridge=$1
nic=$2
- ips=$(ip addr show $bridge | grep 'inet ' | awk -F' ' '{print $2}')
- routes=$(ip route show | grep $bridge)
+ install_gw="$3"
+
+ brige_info=$(ip addr show $bridge 2>/dev/null)
+ if [[ -z $brige_info ]]; then
+ return
+ fi
+ ips=`echo "$brige_info" | grep 'inet ' | grep -v $install_gw | sed "s/inet //g"`
+
+ routes=$(ip route show | grep $bridge | grep -v $install_gw)
ip link set $bridge down
brctl delbr $bridge
- for ip in $ips; do
- ip addr add $ip dev $nic
- done
+ if [[ -n $ips ]]; then
+ echo "$ips" | while read line; do
+ echo $line | sed "s/$bridge/dev $nic/g" | xargs ip addr add | true
+ done
+ fi
- echo "$routes" | while read line; do
- echo $line | sed "s/$bridge/$nic/g" | xargs ip route add | true
+ if [[ -n $routes ]]; then
+ echo "$routes" | while read line; do
+ echo $line | sed "s/$bridge/$nic/g" | xargs ip route add | true
+ done
+ fi
+}
+
+function get_broadcast_addr()
+{
+ ip=(${1//[!0-9]/ })
+ mask=(${2//[!0-9]/ })
+
+ for i in {0..3}; do
+ num=$((${ip[$i]} | $((${mask[$i]} ^ 255))))
+ if [[ -z $broadcast ]]; then
+ broadcast="$num"
+ else
+ broadcast="$broadcast.$num"
+ fi
done
+ echo ${broadcast}
+}
+
+function get_mask_len()
+{
+ mask=`echo $1 | awk -F'.' '{print ($1*(2^24)+$2*(2^16)+$3*(2^8)+$4)}'`
+ mask_len=`echo "obase=2;${mask}"|bc|awk -F'0' '{print length($1)}'`
+ echo $mask_len
}
function create_bridge()
{
bridge=$1
nic=$2
- ips=$(ip addr show $nic | grep 'inet ' | awk -F' ' '{print $2}')
+ ips=$(ip addr show $nic | grep 'inet ' | sed "s/inet //g")
routes=$(ip route show | grep $nic)
ip addr flush $nic
@@ -37,21 +71,25 @@ function create_bridge()
brctl addif $bridge $nic
ip link set $bridge up
- for ip in $ips; do
- ip addr add $ip dev $bridge
- done
+ if [[ -n $ips ]]; then
+ echo "$ips" | while read line; do
+ echo $line | sed -e "s/$nic/dev $bridge/g" | xargs ip addr add
+ done
+ fi
- mask=`echo $INSTALL_MASK | awk -F'.' '{print ($1*(2^24)+$2*(2^16)+$3*(2^8)+$4)}'`
- mask_len=`echo "obase=2;${mask}"|bc|awk -F'0' '{print length($1)}'`
- ip addr add $INSTALL_GW/$mask_len dev $bridge
+ mask_len=`get_mask_len $INSTALL_MASK`
+ broadcast=`get_broadcast_addr $INSTALL_GW $INSTALL_MASK`
+ ip addr add $INSTALL_GW/$mask_len brd $broadcast dev $bridge
- echo "$routes" | while read line; do
- echo $line | sed "s/$nic/$bridge/g" | xargs ip route add | true
- done
+ if [[ -n $routes ]]; then
+ echo "$routes" | while read line; do
+ echo $line | sed "s/$nic/$bridge/g" | xargs ip route add | true
+ done
+ fi
}
function setup_om_bridge() {
- destroy_bridge br_install $OM_NIC
+ destroy_bridge br_install $OM_NIC $INSTALL_GW
create_bridge br_install $OM_NIC
}