diff options
Diffstat (limited to 'deploy')
15 files changed, 369 insertions, 19 deletions
diff --git a/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml index af0727c7..692650b5 100644 --- a/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml +++ b/deploy/adapters/ansible/openstack_juno/HA-ansible-multinodes.yml @@ -53,4 +53,5 @@ remote_user: root sudo: True roles: + - ceph-deploy - monitor diff --git a/deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh b/deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh new file mode 100644 index 00000000..2535f364 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/files/create_osd.sh @@ -0,0 +1,41 @@ +if [ -d "/var/local/osd" ]; then +echo "clear /var/local/osd" +rm -r /var/local/osd/ +umount /var/local/osd +rm -r /var/local/osd +fi + +if [ ! -d "/ceph/images" ]; then +mkdir -p /ceph/images +fi + +rm /ceph/images/ceph-volumes.img + +if [ ! -f "/ceph/images/ceph-volumes.img" ]; then +echo "create ceph-volumes.img" +dd if=/dev/zero of=/ceph/images/ceph-volumes.img bs=1M seek=12288 count=0 oflag=direct +sgdisk -g --clear /ceph/images/ceph-volumes.img +fi + +if [ -L "/dev/ceph-volumes/ceph0" ]; then +echo "remove lv vg" +lvremove /dev/ceph-volumes/ceph0 +vgremove ceph-volumes +rm -r /dev/ceph-volumes +fi + +losetup -d /dev/loop0 + +echo "vgcreate" +vgcreate ceph-volumes $(sudo losetup --show -f /ceph/images/ceph-volumes.img) +echo "lvcreate" +sudo lvcreate -L9G -nceph0 ceph-volumes +echo "mkfs" +mkfs.xfs -f /dev/ceph-volumes/ceph0 + +if [ ! -d "/var/local/osd" ]; then +echo "mount osd" +mkdir -p /var/local/osd +mount /dev/ceph-volumes/ceph0 /var/local/osd +fi + diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml new file mode 100644 index 00000000..f897c944 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_install.yml @@ -0,0 +1,49 @@ +--- +- name: create ceph cluster + shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy new {{ ceph_cluster_hosts.stdout_lines[0] }} + tags: + - create_ceph_cluster + +- name: install ceph for every nodes includes jumpserver + shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy install {{ ceph_cluster_hosts.stdout_lines[0]}} + +- name: create monitor node in controller group + shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy --overwrite-conf mon create-initial + +- name: copy create_osd.sh to host1 + copy: src=create_osd.sh dest=~/create_osd.sh mode=0777 + tags: + - create_osd + +- name: copy create_osd.sh to other nodes + shell: scp -o StrictHostKeyChecking=no ~/create_osd.sh {{ item }}:~/ + with_items: + - "{{ groups['controller'] }}" + tags: + - create_osd + +- name: create osd + shell: ssh -o StrictHostKeyChecking=no -t {{ item }} "~/create_osd.sh" + with_items: + - "{{ groups['controller'] }}" + tags: + - create_osd + +- name: prepare create osd + shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy osd prepare {{ item }}:/var/local/osd + with_items: + - "{{ groups['controller'] }}" + tags: + - create_osd + + +- name: activate osd + shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy osd activate {{ item }}:/var/local/osd && ceph-deploy osd activate {{ item }}:/var/local/osd + with_items: + - "{{ groups['controller'] }}" + tags: + - create_osd + - activate_osd + +- name: create admin node + shell: cd {{ ceph_cluster_dir[0] }} && ceph-deploy admin {{ ceph_cluster_hosts.stdout_lines[0] }} diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml new file mode 100644 index 00000000..34ce6fdc --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_conf.yml @@ -0,0 +1,22 @@ +--- + +- name: modify glance-api.conf for ceph + shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(default_store\).*/\1 = rbd/' /etc/glance/glance-api.conf && sed -i '/^\[glance_store/a stores = rbd\nrbd_store_pool = images\nrbd_store_user = glance\nrbd_store_ceph_conf = /etc/ceph/ceph.conf\nrbd_store_chunk_size = 8' /etc/glance/glance-api.conf && sudo glance-control api restart" + with_items: + - "{{ groups['controller'] }}" + tags: + - ceph_conf_glance + +- name: modify cinder.conf for ceph + shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(volume_driver\).*/\1 = cinder.volume.drivers.rbd.RBDDriver/' /etc/cinder/cinder.conf && sed -i '/^\[DEFAULT/a rbd_pool = volumes\nrbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_flatten_volume_from_snapshot = false\nrbd_max_clone_depth = 5\nrbd_store_chunk_size = 4\nrados_connect_timeout = -1\nglance_api_version = 2\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}' /etc/cinder/cinder.conf && sudo service cinder-volume restart" + with_items: + - "{{ groups['compute'] }}" + tags: + - ceph_conf_cinder + +- name: modify nova.conf for ceph + shell: ssh -o StrictHostKeyChecking=no {{ item }} -t "sed -i 's/^\(images_type\).*/\1 = rbd/' /etc/nova/nova-compute.conf && sed -i '/^\[libvirt/a images_rbd_pool = vms\nimages_rbd_ceph_conf = /etc/ceph/ceph.conf\nrbd_user = cinder\nrbd_secret_uuid = {{ ceph_uuid.stdout_lines[0] }}\ndisk_cachemodes=\"network=writeback\"\nlive_migration_flag=\"VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED\"' /etc/nova/nova-compute.conf && sudo service nova-compute restart" + with_items: + - "{{ groups['compute'] }}" + tags: + - ceph_conf_nova diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml new file mode 100644 index 00000000..41c24709 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_openstack_pre.yml @@ -0,0 +1,59 @@ +--- + +- name: create pool + shell: ceph osd pool create {{ item }} 128 + with_items: + - volumes + - images + - backups + - vms + +- name: create ceph users for openstack + shell: ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images' && ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images' + +- name: send glance key to controller nodes + shell: ceph auth get-or-create client.glance | ssh {{ item }} sudo tee /etc/ceph/ceph.client.glance.keyring && ssh {{ item }} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring + with_items: + - "{{ groups['controller'] }}" + +- name: send cinder key to compute nodes + shell: ceph auth get-or-create client.cinder | ssh {{ item }} sudo tee /etc/ceph/ceph.client.cinder.keyring && ssh {{ item }} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring + with_items: + - "{{ groups['compute'] }}" + tags: + - ceph_send_key + +- name: copy cinder key to compute node + shell: ceph auth get-key client.cinder | ssh {{ item }} tee client.cinder.key + with_items: + - "{{ groups['compute'] }}" + tags: + - ceph_copy_secret + +- name: gen ceph uuid + shell: uuidgen + register: ceph_uuid + tags: + - ceph_copy_secret + +- name: gen template secret.xml + template: src=secret.j2 dest=~/secret.xml mode=0777 + tags: + - ceph_copy_secret + +- name: copy secret.xml to compute nodes + shell: scp -o StrictHostKeyChecking=no ~/secret.xml {{ item }}:~/secret.xml + with_items: + - "{{ groups['compute'] }}" + tags: + - ceph_copy_secret + +- name: create key for libvirt on compute nodes + shell: ssh -o StrictHostKeyChecking=no -t {{ item }} "virsh secret-define --file secret.xml && virsh secret-set-value --secret {{ ceph_uuid.stdout_lines[0] }} --base64 \$(cat client.cinder.key) && rm client.cinder.key secret.xml" + with_items: + - "{{ groups['compute'] }}" + tags: + - ceph_copy_secret + + + diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml new file mode 100644 index 00000000..cacc48d8 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/ceph_setup_env.yml @@ -0,0 +1,51 @@ +--- +- include_vars: "{{ ansible_os_family }}.yml" + +- name: copy rsa secret key from compass-core to host1 + copy: src=/root/.ssh/id_rsa dest=/root/.ssh/id_rsa mode=0600 + +- name: copy rsa pub key from compass-core to host1 + copy: src=/root/.ssh/id_rsa.pub dest=/root/.ssh/id_rsa.pub mode=0644 + +- name: gen ssh key for no-ask access to every nodes + shell: if [ ! -f ~/.ssh/id_rsa.pub ]; then ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N ""; else echo "already gen ssh key!"; fi; + +- name: copy key to node + shell: ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no root@{{ item }} + with_items: + - "{{ groups['all'] }}" + +- name: install ceph-related packages + action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" + with_items: packages | union(packages_noarch) + +- name: clear cluster_hosts_name + file: path=cluster_hosts_name state=absent + tags: + - create_ceph_cluster + - ceph_install + +- name: echo hosts name to file + shell: echo -n "{{ item }} " >> cluster_hosts_name + with_items: + - "{{ groups['all'] }}" + tags: + - create_ceph_cluster + - ceph_install + +- name: register hosts name + shell: cat cluster_hosts_name + register: ceph_cluster_hosts + tags: + - create_ceph_cluster + - echo_hosts + - ceph_install + +- name: gen clean ceph script + template: src=clean_ceph.j2 dest=~/clean_ceph.sh mode=0777 + +- name: clean ceph data of last deploy + shell: ~/clean_ceph.sh + +- name: mkdir for ceph-cluster + file: path={{ ceph_cluster_dir[0] }} state=directory mode=0755 diff --git a/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml b/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml new file mode 100644 index 00000000..a1a9127d --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- include: ceph_setup_env.yml + when: inventory_hostname == "{{ groups['controller'][0] }}" + tags: + - ceph_deploy + - ceph_setup_env + - ceph_install + +- include: ceph_install.yml + when: inventory_hostname == "{{ groups['controller'][0] }}" + tags: + - ceph_deploy + - ceph_install + +- include: ceph_openstack_pre.yml + when: inventory_hostname == "{{ groups['controller'][0] }}" + tags: + - ceph_deploy + - ceph_openstack_pre + - ceph_openstack + +- include: ceph_openstack_conf.yml + when: inventory_hostname == "{{ groups['controller'][0] }}" + tags: + - ceph_deploy + - ceph_openstack_conf + - ceph_openstack diff --git a/deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo b/deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo new file mode 100644 index 00000000..e6b7c7d9 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/templates/ceph.repo @@ -0,0 +1,7 @@ +[ceph-noarch] +name=Ceph noarch packages +baseurl=http://ceph.com/rpm-giant/el6/noarch +enabled=1 +gpgcheck=1 +type=rpm-md +gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc diff --git a/deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j2 b/deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j2 new file mode 100644 index 00000000..a089167d --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/templates/clean_ceph.j2 @@ -0,0 +1,8 @@ +if [ -d "{{ ceph_cluster_dir[0] }}" ]; then +cd {{ ceph_cluster_dir[0] }} +ceph-deploy purge {{ ceph_cluster_hosts.stdout_lines[0] }} +ceph-deploy purgedata {{ ceph_cluster_hosts.stdout_lines[0] }} +ceph-deploy forgetkeys +cd .. +rm -r {{ ceph_cluster_dir[0] }} +fi diff --git a/deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j2 b/deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j2 new file mode 100644 index 00000000..a0ffc6e3 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/templates/secret.j2 @@ -0,0 +1,6 @@ +<secret ephemeral='no' private='no'> + <uuid>{{ ceph_uuid.stdout_lines[0] }}</uuid> + <usage type='ceph'> + <name>client.cinder secret</name> + </usage> +</secret> diff --git a/deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml b/deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml new file mode 100644 index 00000000..5ed6cc10 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/vars/Debian.yml @@ -0,0 +1,18 @@ +--- +packages: + - ceph-deploy + - python-flask + - libgoogle-perftools4 + - libleveldb1 + - liblttng-ust0 + - libsnappy1 + - librbd1 + - librados2 + - python-ceph + - ceph + - ceph-mds + - ceph-common + - ceph-fs-common + - gdisk + +services: [] diff --git a/deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml b/deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml new file mode 100644 index 00000000..5ed6cc10 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/vars/RedHat.yml @@ -0,0 +1,18 @@ +--- +packages: + - ceph-deploy + - python-flask + - libgoogle-perftools4 + - libleveldb1 + - liblttng-ust0 + - libsnappy1 + - librbd1 + - librados2 + - python-ceph + - ceph + - ceph-mds + - ceph-common + - ceph-fs-common + - gdisk + +services: [] diff --git a/deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml b/deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml new file mode 100644 index 00000000..71ace4a5 --- /dev/null +++ b/deploy/adapters/ansible/roles/ceph-deploy/vars/main.yml @@ -0,0 +1,5 @@ +--- +packages_noarch: [] + +ceph_cluster_dir: + - /root/ceph-cluster diff --git a/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg index 4b20db03..169182da 100644 --- a/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg +++ b/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg @@ -6,7 +6,7 @@ global group haproxy maxconn 4000 pidfile /var/run/haproxy/haproxy.pid - #log 127.0.0.1 local0 + log 127.0.0.1 local0 tune.bufsize 1000000 stats socket /var/run/haproxy.sock stats timeout 2m diff --git a/deploy/network.sh b/deploy/network.sh index 864ec011..a5344f58 100755 --- a/deploy/network.sh +++ b/deploy/network.sh @@ -8,27 +8,61 @@ function destroy_bridge() { bridge=$1 nic=$2 - ips=$(ip addr show $bridge | grep 'inet ' | awk -F' ' '{print $2}') - routes=$(ip route show | grep $bridge) + install_gw="$3" + + brige_info=$(ip addr show $bridge 2>/dev/null) + if [[ -z $brige_info ]]; then + return + fi + ips=`echo "$brige_info" | grep 'inet ' | grep -v $install_gw | sed "s/inet //g"` + + routes=$(ip route show | grep $bridge | grep -v $install_gw) ip link set $bridge down brctl delbr $bridge - for ip in $ips; do - ip addr add $ip dev $nic - done + if [[ -n $ips ]]; then + echo "$ips" | while read line; do + echo $line | sed "s/$bridge/dev $nic/g" | xargs ip addr add | true + done + fi - echo "$routes" | while read line; do - echo $line | sed "s/$bridge/$nic/g" | xargs ip route add | true + if [[ -n $routes ]]; then + echo "$routes" | while read line; do + echo $line | sed "s/$bridge/$nic/g" | xargs ip route add | true + done + fi +} + +function get_broadcast_addr() +{ + ip=(${1//[!0-9]/ }) + mask=(${2//[!0-9]/ }) + + for i in {0..3}; do + num=$((${ip[$i]} | $((${mask[$i]} ^ 255)))) + if [[ -z $broadcast ]]; then + broadcast="$num" + else + broadcast="$broadcast.$num" + fi done + echo ${broadcast} +} + +function get_mask_len() +{ + mask=`echo $1 | awk -F'.' '{print ($1*(2^24)+$2*(2^16)+$3*(2^8)+$4)}'` + mask_len=`echo "obase=2;${mask}"|bc|awk -F'0' '{print length($1)}'` + echo $mask_len } function create_bridge() { bridge=$1 nic=$2 - ips=$(ip addr show $nic | grep 'inet ' | awk -F' ' '{print $2}') + ips=$(ip addr show $nic | grep 'inet ' | sed "s/inet //g") routes=$(ip route show | grep $nic) ip addr flush $nic @@ -37,21 +71,25 @@ function create_bridge() brctl addif $bridge $nic ip link set $bridge up - for ip in $ips; do - ip addr add $ip dev $bridge - done + if [[ -n $ips ]]; then + echo "$ips" | while read line; do + echo $line | sed -e "s/$nic/dev $bridge/g" | xargs ip addr add + done + fi - mask=`echo $INSTALL_MASK | awk -F'.' '{print ($1*(2^24)+$2*(2^16)+$3*(2^8)+$4)}'` - mask_len=`echo "obase=2;${mask}"|bc|awk -F'0' '{print length($1)}'` - ip addr add $INSTALL_GW/$mask_len dev $bridge + mask_len=`get_mask_len $INSTALL_MASK` + broadcast=`get_broadcast_addr $INSTALL_GW $INSTALL_MASK` + ip addr add $INSTALL_GW/$mask_len brd $broadcast dev $bridge - echo "$routes" | while read line; do - echo $line | sed "s/$nic/$bridge/g" | xargs ip route add | true - done + if [[ -n $routes ]]; then + echo "$routes" | while read line; do + echo $line | sed "s/$nic/$bridge/g" | xargs ip route add | true + done + fi } function setup_om_bridge() { - destroy_bridge br_install $OM_NIC + destroy_bridge br_install $OM_NIC $INSTALL_GW create_bridge br_install $OM_NIC } |