From 974a3c3bf5b71cd6b07c1b4b51044bc535ff4456 Mon Sep 17 00:00:00 2001 From: baigk Date: Wed, 5 Aug 2015 01:31:07 -0400 Subject: support deployment openstack juno with ansible for compass in ubuntu 14.04 JIRA: COMPASS-4 Change-Id: I7adc565325c2a0cbf878b2aea022b31d118748dd Signed-off-by: baigk --- deploy/adapters/ansible/roles/ha/files/galera_chk | 10 ++ deploy/adapters/ansible/roles/ha/files/mysqlchk | 15 +++ deploy/adapters/ansible/roles/ha/files/notify.sh | 4 + deploy/adapters/ansible/roles/ha/handlers/main.yml | 9 ++ deploy/adapters/ansible/roles/ha/tasks/main.yml | 90 ++++++++++++++ .../ansible/roles/ha/templates/failover.j2 | 65 ++++++++++ .../ansible/roles/ha/templates/haproxy.cfg | 133 +++++++++++++++++++++ .../ansible/roles/ha/templates/keepalived.conf | 42 +++++++ deploy/adapters/ansible/roles/ha/vars/Debian.yml | 10 ++ 9 files changed, 378 insertions(+) create mode 100644 deploy/adapters/ansible/roles/ha/files/galera_chk create mode 100644 deploy/adapters/ansible/roles/ha/files/mysqlchk create mode 100644 deploy/adapters/ansible/roles/ha/files/notify.sh create mode 100644 deploy/adapters/ansible/roles/ha/handlers/main.yml create mode 100644 deploy/adapters/ansible/roles/ha/tasks/main.yml create mode 100644 deploy/adapters/ansible/roles/ha/templates/failover.j2 create mode 100644 deploy/adapters/ansible/roles/ha/templates/haproxy.cfg create mode 100644 deploy/adapters/ansible/roles/ha/templates/keepalived.conf create mode 100644 deploy/adapters/ansible/roles/ha/vars/Debian.yml (limited to 'deploy/adapters/ansible/roles/ha') diff --git a/deploy/adapters/ansible/roles/ha/files/galera_chk b/deploy/adapters/ansible/roles/ha/files/galera_chk new file mode 100644 index 00000000..9fd165c0 --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/files/galera_chk @@ -0,0 +1,10 @@ +#! /bin/sh + +code=`mysql -uroot -e "show status" | awk '/Threads_running/{print $2}'` + +if [ "$code"=="1" ] +then + echo "HTTP/1.1 200 OK\r\n" +else + echo "HTTP/1.1 503 Service Unavailable\r\n" +fi diff --git a/deploy/adapters/ansible/roles/ha/files/mysqlchk b/deploy/adapters/ansible/roles/ha/files/mysqlchk new file mode 100644 index 00000000..7c5eaad5 --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/files/mysqlchk @@ -0,0 +1,15 @@ +# default: off +# description: An xinetd internal service which echo's characters back to +# clients. +# This is the tcp version. +service mysqlchk +{ + disable = no + flags = REUSE + socket_type = stream + protocol = tcp + user = root + wait = no + server = /usr/local/bin/galera_chk + port = 9200 +} diff --git a/deploy/adapters/ansible/roles/ha/files/notify.sh b/deploy/adapters/ansible/roles/ha/files/notify.sh new file mode 100644 index 00000000..5edffe84 --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/files/notify.sh @@ -0,0 +1,4 @@ +#!/bin/bash +python /usr/local/bin/failover.py $1 +mysql -uroot -e"flush hosts" +service mysql restart diff --git a/deploy/adapters/ansible/roles/ha/handlers/main.yml b/deploy/adapters/ansible/roles/ha/handlers/main.yml new file mode 100644 index 00000000..a02c686b --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/handlers/main.yml @@ -0,0 +1,9 @@ +--- +- name: restart haproxy + service: name=haproxy state=restarted enabled=yes + +- name: restart xinetd + service: name=xinetd state=restarted enabled=yes + +- name: restart keepalived + service: name=keepalived state=restarted enabled=yes diff --git a/deploy/adapters/ansible/roles/ha/tasks/main.yml b/deploy/adapters/ansible/roles/ha/tasks/main.yml new file mode 100644 index 00000000..eeaf5921 --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/tasks/main.yml @@ -0,0 +1,90 @@ +--- +- include_vars: "{{ ansible_os_family }}.yml" + +- name: install keepalived xinet haproxy + action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" + with_items: packages + +- name: generate ha service list + shell: echo {{ item }} >> /opt/service + with_items: services + +- name: install pexpect + pip: name=pexpect state=present + +- name: activate ip_nonlocal_bind + sysctl: name=net.ipv4.ip_nonlocal_bind value=1 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_intvl + sysctl: name=net.ipv4.tcp_keepalive_intvl value=1 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_probes + sysctl: name=net.ipv4.tcp_keepalive_probes value=5 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_time + sysctl: name=net.ipv4.tcp_keepalive_time value=5 + state=present reload=yes + +- name: update haproxy cfg + template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg + notify: restart haproxy + +- name: set haproxy enable flag + lineinfile: dest=/etc/default/haproxy state=present + regexp="ENABLED=*" + line="ENABLED=1" + notify: restart haproxy + +- name: set haproxy log + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="local0.* /var/log/haproxy.log" + line="local0.* /var/log/haproxy.log" + +- name: set rsyslog udp module + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="^#$ModLoad imudp" + line="$ModLoad imudp" + +- name: set rsyslog udp port + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="^#$UDPServerRun 514" + line="$UDPServerRun 514" + +- name: copy galera_chk file + copy: src=galera_chk dest=/usr/local/bin/galera_chk mode=0777 + +- name: copy notify file + copy: src=notify.sh dest=/usr/local/bin/notify.sh mode=0777 + +- name: copy notify template file + template: src=failover.j2 dest=/usr/local/bin/failover.py mode=0777 + +- name: add network service + lineinfile: dest=/etc/services state=present + line="mysqlchk 9200/tcp" + insertafter="Local services" + notify: restart xinetd + +- name: copy mysqlchk file + copy: src=mysqlchk dest=/etc/xinetd.d/mysqlchk mode=0777 + notify: restart xinetd + +- name: set keepalived start param + lineinfile: dest=/etc/default/keepalived state=present + regexp="^DAEMON_ARGS=*" + line="DAEMON_ARGS=\"-D -d -S 1\"" + +- name: set keepalived log + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="local1.* /var/log/keepalived.log" + line="local1.* /var/log/keepalived.log" + +- name: update keepalived info + template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf + notify: restart keepalived + +- name: restart rsyslog + shell: service rsyslog restart diff --git a/deploy/adapters/ansible/roles/ha/templates/failover.j2 b/deploy/adapters/ansible/roles/ha/templates/failover.j2 new file mode 100644 index 00000000..e3877e6b --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/templates/failover.j2 @@ -0,0 +1,65 @@ +import ConfigParser, os, socket +import logging as LOG +import pxssh +import sys +import re + +LOG_FILE="/var/log/mysql_failover" +try: + os.remove(LOG_FILE) +except: + pass + +LOG.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename=LOG_FILE,level=LOG.DEBUG) +ha_vip = {{ HA_VIP }} +LOG.info("ha_vip: %s" % ha_vip) + +#ha_vip = "10.1.0.50" +galera_path = '/etc/mysql/conf.d/wsrep.cnf' +pattern = re.compile(r"gcomm://(?P.*)") + +def ssh_get_hostname(ip): + try: + s = pxssh.pxssh() + s.login("%s" % ip, "root", "root") + s.sendline('hostname') # run a command + s.prompt() # match the prompt + result = s.before.strip() # print everything before the prompt. + return result.split(os.linesep)[1] + except pxssh.ExceptionPxssh as e: + LOG.error("pxssh failed on login.") + raise + +def failover(mode): + config = ConfigParser.ConfigParser() + config.optionxform = str + config.readfp(open(galera_path)) + wsrep_cluster_address = config.get("mysqld", "wsrep_cluster_address") + wsrep_cluster_address = pattern.match(wsrep_cluster_address).groupdict()["prev_ip"] + + LOG.info("old wsrep_cluster_address = %s" % wsrep_cluster_address) + + if mode == "master": + # refresh wsrep_cluster_address to null + LOG.info("I'm being master, set wsrep_cluster_address to null") + wsrep_cluster_address = "" + + elif mode == "backup": + # refresh wsrep_cluster_address to master int ip + hostname = ssh_get_hostname(ha_vip) + wsrep_cluster_address = socket.gethostbyname(hostname) + LOG.info("I'm being slave, set wsrep_cluster_address to master internal ip") + + LOG.info("new wsrep_cluster_address = %s" % wsrep_cluster_address) + wsrep_cluster_address = "gcomm://%s" % wsrep_cluster_address + config.set("mysqld", "wsrep_cluster_address", wsrep_cluster_address) + with open(galera_path, 'wb') as fp: + #config.write(sys.stdout) + config.write(fp) + + os.system("service mysql restart") + LOG.info("failover success!!!") + +if __name__ == "__main__": + LOG.debug("call me: %s" % sys.argv) + failover(sys.argv[1]) diff --git a/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg new file mode 100644 index 00000000..4ed528ad --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/templates/haproxy.cfg @@ -0,0 +1,133 @@ + +global + #chroot /var/run/haproxy + daemon + user haproxy + group haproxy + maxconn 4000 + pidfile /var/run/haproxy/haproxy.pid + #log 127.0.0.1 local0 + tune.bufsize 1000000 + stats socket /var/run/haproxy.sock + stats timeout 2m + +defaults + log global + maxconn 8000 + option redispatch + option dontlognull + option splice-auto + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 6m + timeout server 6m + timeout check 10s + retries 5 + +listen proxy-glance_registry_cluster + bind {{ HA_VIP }}:9191 + option tcpka + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9191 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-glance_api_cluster + bind {{ HA_VIP }}:9292 + option tcpka + option httpchk + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9292 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-nova-novncproxy + bind {{ HA_VIP }}:6080 + option tcpka + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:6080 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-network + bind {{ HA_VIP }}:9696 + option tcpka + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9696 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-volume + bind {{ HA_VIP }}:8776 + option tcpka + option httpchk + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8776 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-keystone_admin_cluster + bind {{ HA_VIP }}:35357 + option tcpka + option httpchk + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:35357 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-keystone_public_internal_cluster + bind {{ HA_VIP }}:5000 + option tcpka + option httpchk + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:5000 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-nova_compute_api_cluster + bind {{ HA_VIP }}:8774 + mode tcp + option httpchk + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8774 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-nova_metadata_api_cluster + bind {{ HA_VIP }}:8775 + option tcpka + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8775 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen proxy-cinder_api_cluster + bind {{ HA_VIP }}:8776 + mode tcp + option httpchk + option tcplog + balance source +{% for host in groups['controller'] %} + server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8776 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen stats + mode http + bind 0.0.0.0:8888 + stats enable + stats refresh 30s + stats uri / + stats realm Global\ statistics + stats auth admin:admin + + diff --git a/deploy/adapters/ansible/roles/ha/templates/keepalived.conf b/deploy/adapters/ansible/roles/ha/templates/keepalived.conf new file mode 100644 index 00000000..02c55326 --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/templates/keepalived.conf @@ -0,0 +1,42 @@ +global_defs { + + notification_email{ + root@huawei.com + } + + notification_email_from keepalived@huawei.com + + smtp_server localhost + + smtp_connect_timeout 30 + + router_id NodeA + +} + +vrrp_instance VI_1 { + + interface {{ INTERNAL_INTERFACE }} + virtual_router_id 51 + state BACKUP + nopreempt + advert_int 1 +{% for host in groups['controller'] %} +{% if host == inventory_hostname %} + priority {{ 100 - loop.index0 * 5 }} +{% endif %} +{% endfor %} + + authentication { + auth_type PASS + auth_pass 1111 + } + + virtual_ipaddress { + {{ HA_VIP }} dev {{ INTERNAL_INTERFACE }} + } + + notify_master "/usr/local/bin/notify.sh master" + notify_backup "/usr/local/bin/notify.sh backup" +} + diff --git a/deploy/adapters/ansible/roles/ha/vars/Debian.yml b/deploy/adapters/ansible/roles/ha/vars/Debian.yml new file mode 100644 index 00000000..263c2ea7 --- /dev/null +++ b/deploy/adapters/ansible/roles/ha/vars/Debian.yml @@ -0,0 +1,10 @@ +--- +packages: + - keepalived + - xinetd + - haproxy + +services: + - keepalived + - xinetd + - haproxy -- cgit 1.2.3-korg