summaryrefslogtreecommitdiffstats
path: root/compass/deploy/ansible/roles/ha
diff options
context:
space:
mode:
Diffstat (limited to 'compass/deploy/ansible/roles/ha')
-rw-r--r--compass/deploy/ansible/roles/ha/files/galera_chk10
-rw-r--r--compass/deploy/ansible/roles/ha/files/mysqlchk15
-rw-r--r--compass/deploy/ansible/roles/ha/files/notify.sh4
-rw-r--r--compass/deploy/ansible/roles/ha/handlers/main.yml9
-rw-r--r--compass/deploy/ansible/roles/ha/tasks/main.yml94
-rw-r--r--compass/deploy/ansible/roles/ha/templates/failover.j265
-rw-r--r--compass/deploy/ansible/roles/ha/templates/haproxy.cfg133
-rw-r--r--compass/deploy/ansible/roles/ha/templates/keepalived.conf42
8 files changed, 372 insertions, 0 deletions
diff --git a/compass/deploy/ansible/roles/ha/files/galera_chk b/compass/deploy/ansible/roles/ha/files/galera_chk
new file mode 100644
index 000000000..9fd165c00
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/files/galera_chk
@@ -0,0 +1,10 @@
+#! /bin/sh
+
+code=`mysql -uroot -e "show status" | awk '/Threads_running/{print $2}'`
+
+if [ "$code"=="1" ]
+then
+ echo "HTTP/1.1 200 OK\r\n"
+else
+ echo "HTTP/1.1 503 Service Unavailable\r\n"
+fi
diff --git a/compass/deploy/ansible/roles/ha/files/mysqlchk b/compass/deploy/ansible/roles/ha/files/mysqlchk
new file mode 100644
index 000000000..2c03f19f4
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/files/mysqlchk
@@ -0,0 +1,15 @@
+# default: off
+# description: An xinetd internal service which echo's characters back to
+# clients.
+# This is the tcp version.
+service mysqlchk
+{
+ disable = no
+ flags = REUSE
+ socket_type = stream
+ protocol = tcp
+ user = root
+ wait = no
+ server = /usr/local/bin/galera_chk
+ port = 9200
+}
diff --git a/compass/deploy/ansible/roles/ha/files/notify.sh b/compass/deploy/ansible/roles/ha/files/notify.sh
new file mode 100644
index 000000000..5edffe84b
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/files/notify.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+python /usr/local/bin/failover.py $1
+mysql -uroot -e"flush hosts"
+service mysql restart
diff --git a/compass/deploy/ansible/roles/ha/handlers/main.yml b/compass/deploy/ansible/roles/ha/handlers/main.yml
new file mode 100644
index 000000000..a02c686bf
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/handlers/main.yml
@@ -0,0 +1,9 @@
+---
+- name: restart haproxy
+ service: name=haproxy state=restarted enabled=yes
+
+- name: restart xinetd
+ service: name=xinetd state=restarted enabled=yes
+
+- name: restart keepalived
+ service: name=keepalived state=restarted enabled=yes
diff --git a/compass/deploy/ansible/roles/ha/tasks/main.yml b/compass/deploy/ansible/roles/ha/tasks/main.yml
new file mode 100644
index 000000000..a00c21ac4
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/tasks/main.yml
@@ -0,0 +1,94 @@
+---
+- name: install keepalived xinet haproxy
+ apt: name={{ item }} state=present
+ with_items:
+ - keepalived
+ - xinetd
+ - haproxy
+
+- name: generate ha service list
+ shell: echo {{ item }} >> /opt/service
+ with_items:
+ - keepalived
+ - xinetd
+ - haproxy
+
+- name: install pexpect
+ pip: name=pexpect state=present
+
+- name: activate ip_nonlocal_bind
+ sysctl: name=net.ipv4.ip_nonlocal_bind value=1
+ state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_intvl
+ sysctl: name=net.ipv4.tcp_keepalive_intvl value=1
+ state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_probes
+ sysctl: name=net.ipv4.tcp_keepalive_probes value=5
+ state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_time
+ sysctl: name=net.ipv4.tcp_keepalive_time value=5
+ state=present reload=yes
+
+- name: update haproxy cfg
+ template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg
+ notify: restart haproxy
+
+- name: set haproxy enable flag
+ lineinfile: dest=/etc/default/haproxy state=present
+ regexp="ENABLED=*"
+ line="ENABLED=1"
+ notify: restart haproxy
+
+- name: set haproxy log
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="local0.* /var/log/haproxy.log"
+ line="local0.* /var/log/haproxy.log"
+
+- name: set rsyslog udp module
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="^#$ModLoad imudp"
+ line="$ModLoad imudp"
+
+- name: set rsyslog udp port
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="^#$UDPServerRun 514"
+ line="$UDPServerRun 514"
+
+- name: copy galera_chk file
+ copy: src=galera_chk dest=/usr/local/bin/galera_chk mode=0777
+
+- name: copy notify file
+ copy: src=notify.sh dest=/usr/local/bin/notify.sh mode=0777
+
+- name: copy notify template file
+ template: src=failover.j2 dest=/usr/local/bin/failover.py mode=0777
+
+- name: add network service
+ lineinfile: dest=/etc/services state=present
+ line="mysqlchk 9200/tcp"
+ insertafter="Local services"
+ notify: restart xinetd
+
+- name: copy mysqlchk file
+ copy: src=mysqlchk dest=/etc/xinetd.d/mysqlchk mode=0777
+ notify: restart xinetd
+
+- name: set keepalived start param
+ lineinfile: dest=/etc/default/keepalived state=present
+ regexp="^DAEMON_ARGS=*"
+ line="DAEMON_ARGS=\"-D -d -S 1\""
+
+- name: set keepalived log
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="local1.* /var/log/keepalived.log"
+ line="local1.* /var/log/keepalived.log"
+
+- name: update keepalived info
+ template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf
+ notify: restart keepalived
+
+- name: restart rsyslog
+ shell: service rsyslog restart
diff --git a/compass/deploy/ansible/roles/ha/templates/failover.j2 b/compass/deploy/ansible/roles/ha/templates/failover.j2
new file mode 100644
index 000000000..b03c7375d
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/templates/failover.j2
@@ -0,0 +1,65 @@
+import ConfigParser, os, socket
+import logging as LOG
+import pxssh
+import sys
+import re
+
+LOG_FILE="/var/log/mysql_failover"
+try:
+ os.remove(LOG_FILE)
+except:
+ pass
+
+LOG.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename=LOG_FILE,level=LOG.DEBUG)
+ha_vip = {{ HA_VIP }}
+LOG.info("ha_vip: %s" % ha_vip)
+
+#ha_vip = "10.1.0.50"
+galera_path = '/etc/mysql/conf.d/wsrep.cnf'
+pattern = re.compile(r"gcomm://(?P<prev_ip>.*)")
+
+def ssh_get_hostname(ip):
+ try:
+ s = pxssh.pxssh()
+ s.login("%s" % ip, "root", "root")
+ s.sendline('hostname') # run a command
+ s.prompt() # match the prompt
+ result = s.before.strip() # print everything before the prompt.
+ return result.split(os.linesep)[1]
+ except pxssh.ExceptionPxssh as e:
+ LOG.error("pxssh failed on login.")
+ raise
+
+def failover(mode):
+ config = ConfigParser.ConfigParser()
+ config.optionxform = str
+ config.readfp(open(galera_path))
+ wsrep_cluster_address = config.get("mysqld", "wsrep_cluster_address")
+ wsrep_cluster_address = pattern.match(wsrep_cluster_address).groupdict()["prev_ip"]
+
+ LOG.info("old wsrep_cluster_address = %s" % wsrep_cluster_address)
+
+ if mode == "master":
+ # refresh wsrep_cluster_address to null
+ LOG.info("I'm being master, set wsrep_cluster_address to null")
+ wsrep_cluster_address = ""
+
+ elif mode == "backup":
+ # refresh wsrep_cluster_address to master int ip
+ hostname = ssh_get_hostname(ha_vip)
+ wsrep_cluster_address = socket.gethostbyname(hostname)
+ LOG.info("I'm being slave, set wsrep_cluster_address to master internal ip")
+
+ LOG.info("new wsrep_cluster_address = %s" % wsrep_cluster_address)
+ wsrep_cluster_address = "gcomm://%s" % wsrep_cluster_address
+ config.set("mysqld", "wsrep_cluster_address", wsrep_cluster_address)
+ with open(galera_path, 'wb') as fp:
+ #config.write(sys.stdout)
+ config.write(fp)
+
+ os.system("service mysql restart")
+ LOG.info("failover success!!!")
+
+if __name__ == "__main__":
+ LOG.debug("call me: %s" % sys.argv)
+ failover(sys.argv[1])
diff --git a/compass/deploy/ansible/roles/ha/templates/haproxy.cfg b/compass/deploy/ansible/roles/ha/templates/haproxy.cfg
new file mode 100644
index 000000000..4ed528ad6
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/templates/haproxy.cfg
@@ -0,0 +1,133 @@
+
+global
+ #chroot /var/run/haproxy
+ daemon
+ user haproxy
+ group haproxy
+ maxconn 4000
+ pidfile /var/run/haproxy/haproxy.pid
+ #log 127.0.0.1 local0
+ tune.bufsize 1000000
+ stats socket /var/run/haproxy.sock
+ stats timeout 2m
+
+defaults
+ log global
+ maxconn 8000
+ option redispatch
+ option dontlognull
+ option splice-auto
+ timeout http-request 10s
+ timeout queue 1m
+ timeout connect 10s
+ timeout client 6m
+ timeout server 6m
+ timeout check 10s
+ retries 5
+
+listen proxy-glance_registry_cluster
+ bind {{ HA_VIP }}:9191
+ option tcpka
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9191 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-glance_api_cluster
+ bind {{ HA_VIP }}:9292
+ option tcpka
+ option httpchk
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9292 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-nova-novncproxy
+ bind {{ HA_VIP }}:6080
+ option tcpka
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:6080 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-network
+ bind {{ HA_VIP }}:9696
+ option tcpka
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9696 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-volume
+ bind {{ HA_VIP }}:8776
+ option tcpka
+ option httpchk
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8776 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-keystone_admin_cluster
+ bind {{ HA_VIP }}:35357
+ option tcpka
+ option httpchk
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:35357 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-keystone_public_internal_cluster
+ bind {{ HA_VIP }}:5000
+ option tcpka
+ option httpchk
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:5000 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-nova_compute_api_cluster
+ bind {{ HA_VIP }}:8774
+ mode tcp
+ option httpchk
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8774 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-nova_metadata_api_cluster
+ bind {{ HA_VIP }}:8775
+ option tcpka
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8775 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen proxy-cinder_api_cluster
+ bind {{ HA_VIP }}:8776
+ mode tcp
+ option httpchk
+ option tcplog
+ balance source
+{% for host in groups['controller'] %}
+ server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8776 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen stats
+ mode http
+ bind 0.0.0.0:8888
+ stats enable
+ stats refresh 30s
+ stats uri /
+ stats realm Global\ statistics
+ stats auth admin:admin
+
+
diff --git a/compass/deploy/ansible/roles/ha/templates/keepalived.conf b/compass/deploy/ansible/roles/ha/templates/keepalived.conf
new file mode 100644
index 000000000..0b491376e
--- /dev/null
+++ b/compass/deploy/ansible/roles/ha/templates/keepalived.conf
@@ -0,0 +1,42 @@
+global_defs {
+
+ notification_email{
+ root@huawei.com
+ }
+
+ notification_email_from keepalived@huawei.com
+
+ smtp_server localhost
+
+ smtp_connect_timeout 30
+
+ router_id NodeA
+
+}
+
+vrrp_instance VI_1 {
+
+ interface {{ INTERNAL_INTERFACE }}
+ virtual_router_id 51
+ state BACKUP
+ nopreempt
+ advert_int 1
+{% for host in groups['controller'] %}
+{% if host == inventory_hostname %}
+ priority {{ 100 - loop.index0 * 5 }}
+{% endif %}
+{% endfor %}
+
+ authentication {
+ auth_type PASS
+ auth_pass 1111
+ }
+
+ virtual_ipaddress {
+ {{ HA_VIP }} dev {{ INTERNAL_INTERFACE }}
+ }
+
+ notify_master "/usr/local/bin/notify.sh master"
+ notify_backup "/usr/local/bin/notify.sh backup"
+}
+