Add openstack HA installer code with ansible for compass adapter

patch 2: add service monitor and adjust monitor timelen

BGS-25

Change-Id: I4239576b3888a0a7f6b697fe7fffffb677ef7dab
Signed-off-by: baigk <baiguoku@huawei.com>

8 files changed, 372 insertions, 0 deletions

diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/files/galera_chk b/compass/deploy/ansible/openstack_juno/roles/ha/files/galera_chk
new file mode 100644
index 0000000..9fd165c
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/files/galera_chk
@@ -0,0 +1,10 @@
+#! /bin/sh
+
+code=`mysql -uroot -e "show status" | awk '/Threads_running/{print $2}'`
+
+if [ "$code"=="1" ]
+then
+    echo "HTTP/1.1 200 OK\r\n"
+else
+    echo "HTTP/1.1 503 Service Unavailable\r\n"
+fi
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/files/mysqlchk b/compass/deploy/ansible/openstack_juno/roles/ha/files/mysqlchk
new file mode 100644
index 0000000..2c03f19
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/files/mysqlchk
@@ -0,0 +1,15 @@
+# default: off
+# description: An xinetd internal service which echo's characters back to
+# clients.
+# This is the tcp version.
+service mysqlchk
+{
+    disable     = no
+    flags       = REUSE
+    socket_type = stream
+    protocol    = tcp
+    user        = root
+    wait        = no
+    server      = /usr/local/bin/galera_chk
+    port        = 9200
+}    
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/files/notify.sh b/compass/deploy/ansible/openstack_juno/roles/ha/files/notify.sh
new file mode 100644
index 0000000..5edffe8
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/files/notify.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+python /usr/local/bin/failover.py $1
+mysql -uroot -e"flush hosts"
+service mysql restart
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/handlers/main.yml b/compass/deploy/ansible/openstack_juno/roles/ha/handlers/main.yml
new file mode 100644
index 0000000..a02c686
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/handlers/main.yml
@@ -0,0 +1,9 @@
+---
+- name: restart haproxy
+  service: name=haproxy state=restarted enabled=yes
+
+- name: restart xinetd
+  service: name=xinetd state=restarted enabled=yes
+
+- name: restart keepalived
+  service: name=keepalived state=restarted enabled=yes
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/tasks/main.yml b/compass/deploy/ansible/openstack_juno/roles/ha/tasks/main.yml
new file mode 100644
index 0000000..a00c21a
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/tasks/main.yml
@@ -0,0 +1,94 @@
+---
+- name: install keepalived xinet haproxy 
+  apt: name={{ item }} state=present
+  with_items:
+    - keepalived
+    - xinetd
+    - haproxy
+   
+- name: generate ha service list
+  shell: echo {{ item }} >> /opt/service
+  with_items:
+    - keepalived
+    - xinetd
+    - haproxy
+
+- name: install pexpect
+  pip: name=pexpect state=present
+
+- name: activate ip_nonlocal_bind
+  sysctl: name=net.ipv4.ip_nonlocal_bind value=1
+          state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_intvl 
+  sysctl: name=net.ipv4.tcp_keepalive_intvl value=1
+          state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_probes
+  sysctl: name=net.ipv4.tcp_keepalive_probes value=5
+          state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_time
+  sysctl: name=net.ipv4.tcp_keepalive_time value=5
+          state=present reload=yes
+
+- name: update haproxy cfg
+  template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg
+  notify: restart haproxy
+
+- name: set haproxy enable flag
+  lineinfile: dest=/etc/default/haproxy state=present
+              regexp="ENABLED=*"
+              line="ENABLED=1"
+  notify: restart haproxy
+
+- name: set haproxy log
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="local0.* /var/log/haproxy.log"
+              line="local0.* /var/log/haproxy.log"
+
+- name: set rsyslog udp module
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="^#$ModLoad imudp"
+              line="$ModLoad imudp"
+
+- name: set rsyslog udp port
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="^#$UDPServerRun 514"
+              line="$UDPServerRun 514"
+
+- name: copy galera_chk file
+  copy: src=galera_chk dest=/usr/local/bin/galera_chk mode=0777
+
+- name: copy notify file
+  copy: src=notify.sh dest=/usr/local/bin/notify.sh mode=0777
+  
+- name: copy notify template file
+  template: src=failover.j2 dest=/usr/local/bin/failover.py mode=0777
+
+- name: add network service
+  lineinfile: dest=/etc/services state=present
+              line="mysqlchk          9200/tcp"
+              insertafter="Local services"
+  notify: restart xinetd
+
+- name: copy mysqlchk file
+  copy: src=mysqlchk dest=/etc/xinetd.d/mysqlchk mode=0777
+  notify: restart xinetd
+
+- name: set keepalived start param
+  lineinfile: dest=/etc/default/keepalived state=present
+              regexp="^DAEMON_ARGS=*"
+              line="DAEMON_ARGS=\"-D -d -S 1\""
+
+- name: set keepalived log
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="local1.* /var/log/keepalived.log"
+              line="local1.* /var/log/keepalived.log"
+
+- name: update keepalived info
+  template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf
+  notify: restart keepalived
+
+- name: restart rsyslog
+  shell: service rsyslog restart
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/templates/failover.j2 b/compass/deploy/ansible/openstack_juno/roles/ha/templates/failover.j2
new file mode 100644
index 0000000..b03c737
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/templates/failover.j2
@@ -0,0 +1,65 @@
+import ConfigParser, os, socket
+import logging as LOG
+import pxssh
+import sys
+import re
+
+LOG_FILE="/var/log/mysql_failover"
+try:
+    os.remove(LOG_FILE)
+except:
+    pass
+
+LOG.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename=LOG_FILE,level=LOG.DEBUG)
+ha_vip = {{ HA_VIP }} 
+LOG.info("ha_vip: %s" % ha_vip)
+
+#ha_vip = "10.1.0.50"
+galera_path = '/etc/mysql/conf.d/wsrep.cnf'
+pattern = re.compile(r"gcomm://(?P<prev_ip>.*)")
+
+def ssh_get_hostname(ip):
+    try:
+        s = pxssh.pxssh()
+        s.login("%s" % ip, "root", "root")
+        s.sendline('hostname')   # run a command
+        s.prompt()             # match the prompt
+        result = s.before.strip()      # print everything before the prompt.
+        return result.split(os.linesep)[1]
+    except pxssh.ExceptionPxssh as e:
+        LOG.error("pxssh failed on login.")
+	raise
+
+def failover(mode):
+    config = ConfigParser.ConfigParser()
+    config.optionxform = str
+    config.readfp(open(galera_path))
+    wsrep_cluster_address = config.get("mysqld", "wsrep_cluster_address")
+    wsrep_cluster_address = pattern.match(wsrep_cluster_address).groupdict()["prev_ip"]
+
+    LOG.info("old wsrep_cluster_address = %s" % wsrep_cluster_address)
+
+    if mode == "master":
+        # refresh wsrep_cluster_address to null
+        LOG.info("I'm being master, set wsrep_cluster_address to null")
+	wsrep_cluster_address = ""
+
+    elif mode == "backup":
+        # refresh wsrep_cluster_address to master int ip
+	hostname = ssh_get_hostname(ha_vip) 
+	wsrep_cluster_address = socket.gethostbyname(hostname)
+        LOG.info("I'm being slave, set wsrep_cluster_address to master internal ip")
+
+    LOG.info("new wsrep_cluster_address = %s" % wsrep_cluster_address)
+    wsrep_cluster_address  = "gcomm://%s" % wsrep_cluster_address
+    config.set("mysqld", "wsrep_cluster_address", wsrep_cluster_address)
+    with open(galera_path, 'wb') as fp:
+        #config.write(sys.stdout)
+        config.write(fp)
+   
+    os.system("service mysql restart")
+    LOG.info("failover success!!!")
+
+if __name__ == "__main__":
+    LOG.debug("call me: %s" % sys.argv)
+    failover(sys.argv[1])
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/templates/haproxy.cfg b/compass/deploy/ansible/openstack_juno/roles/ha/templates/haproxy.cfg
new file mode 100644
index 0000000..4ed528a
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/templates/haproxy.cfg
@@ -0,0 +1,133 @@
+
+global
+    #chroot /var/run/haproxy
+    daemon
+    user haproxy
+    group haproxy
+    maxconn 4000
+    pidfile /var/run/haproxy/haproxy.pid
+    #log 127.0.0.1 local0
+    tune.bufsize 1000000
+    stats socket /var/run/haproxy.sock
+    stats timeout 2m
+
+defaults
+    log global
+    maxconn 8000
+    option redispatch
+    option dontlognull
+    option splice-auto
+    timeout http-request 10s
+    timeout queue 1m
+    timeout connect 10s
+    timeout client 6m
+    timeout server 6m
+    timeout check 10s
+    retries 5
+
+listen  proxy-glance_registry_cluster
+    bind {{ HA_VIP }}:9191
+    option tcpka
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9191 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-glance_api_cluster
+    bind {{ HA_VIP }}:9292
+    option tcpka
+    option httpchk
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9292 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-nova-novncproxy
+    bind {{ HA_VIP }}:6080
+    option tcpka
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:6080 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-network
+    bind {{ HA_VIP }}:9696
+    option tcpka
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:9696 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-volume
+    bind {{ HA_VIP }}:8776
+    option tcpka
+    option httpchk
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8776 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-keystone_admin_cluster
+    bind {{ HA_VIP }}:35357
+    option tcpka
+    option httpchk
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:35357 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-keystone_public_internal_cluster
+    bind {{ HA_VIP }}:5000
+    option tcpka
+    option httpchk
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:5000 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-nova_compute_api_cluster
+    bind {{ HA_VIP }}:8774
+    mode tcp
+    option httpchk
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8774 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-nova_metadata_api_cluster
+    bind {{ HA_VIP }}:8775
+    option tcpka
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8775 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen  proxy-cinder_api_cluster
+    bind {{ HA_VIP }}:8776
+    mode tcp
+    option httpchk
+    option tcplog
+    balance source
+{% for host in groups['controller'] %}
+    server {{ host }} {{ hostvars[host]['ansible_' + INTERNAL_INTERFACE].ipv4.address }}:8776 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen stats
+    mode http
+    bind 0.0.0.0:8888
+    stats enable
+    stats refresh 30s
+    stats uri /
+    stats realm Global\ statistics
+    stats auth admin:admin
+
+
diff --git a/compass/deploy/ansible/openstack_juno/roles/ha/templates/keepalived.conf b/compass/deploy/ansible/openstack_juno/roles/ha/templates/keepalived.conf
new file mode 100644
index 0000000..0b49137
--- /dev/null
+++ b/compass/deploy/ansible/openstack_juno/roles/ha/templates/keepalived.conf
@@ -0,0 +1,42 @@
+global_defs {
+
+    notification_email{
+        root@huawei.com
+    }
+
+    notification_email_from keepalived@huawei.com
+
+    smtp_server localhost
+
+    smtp_connect_timeout 30
+
+    router_id  NodeA
+
+}
+
+vrrp_instance VI_1 {
+    
+    interface {{ INTERNAL_INTERFACE }}
+    virtual_router_id 51
+    state BACKUP
+    nopreempt
+    advert_int 1
+{% for host in groups['controller'] %}
+{% if host == inventory_hostname %}
+    priority {{ 100 - loop.index0 * 5 }}
+{% endif %}
+{% endfor %}
+
+    authentication {
+        auth_type PASS
+        auth_pass 1111
+    }
+
+    virtual_ipaddress {
+        {{ HA_VIP }} dev {{ INTERNAL_INTERFACE }}
+    }
+
+    notify_master "/usr/local/bin/notify.sh master"
+    notify_backup "/usr/local/bin/notify.sh backup"
+}
+