bugfix: opencontrail automation deployment scripts

JIRA: COMPASS-277

Change-Id: Ia1f52c6e8e3de520cf06de99550c17879aaed2d7
Signed-off-by: chenshuai@huawei.com <chenshuai@huawei.com>

10 files changed, 233 insertions, 18 deletions

diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-analytics-api-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-analytics-api-conf.j2
index 3ac84c33..dd5e121e 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-analytics-api-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-analytics-api-conf.j2
@@ -2,9 +2,9 @@
 host_ip = {{ contrail_address }}
 rest_api_ip = {{ contrail_haproxy_address }}
 rest_api_port = 8081
-#cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ ip_settings[cur_host]['mgmt']['ip'] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+#cassandra_server_list = {% for cur_host in groups['controller'] %}{{ ip_settings[cur_host]['mgmt']['ip'] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 #chenshuai: This kind of written is also correct, but the following is better, this as record
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 collectors = {{ contrail_haproxy_address }}:8086
 http_server_port = 8090
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-api-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-api-conf.j2
index 79ba9479..65584138 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-api-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-api-conf.j2
@@ -5,9 +5,9 @@ ifmap_server_ip = {{ contrail_haproxy_address }}
 ifmap_server_port = 8443
 ifmap_username = api-server
 ifmap_password = api-server
-zk_server_ip = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
+zk_server_ip = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
 
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 disc_server_ip = {{ contrail_haproxy_address }}
 disc_server_port = 5998
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-collector-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-collector-conf.j2
index da5cf2ab..d4cf2cf6 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-collector-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-collector-conf.j2
@@ -13,7 +13,7 @@ analytics_flow_ttl = -1
 
 # IP address and port to be used to connect to cassandra.
 # Multiple IP:port strings separated by space can be provided
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 
 # IP address and port to be used to connect to kafka.
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-device-manager-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-device-manager-conf.j2
index 2c45d21a..b2b90a50 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-device-manager-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-device-manager-conf.j2
@@ -1,9 +1,9 @@
 [DEFAULTS]
 api_server_ip = {{ contrail_haproxy_address }}
 api_server_port = 8082
-zk_server_ip = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
+zk_server_ip = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
 
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 disc_server_ip = {{ contrail_haproxy_address }}
 disc_server_port = 5998
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-discovery-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-discovery-conf.j2
index 509b3569..d8dbb2c7 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-discovery-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-discovery-conf.j2
@@ -1,10 +1,10 @@
 [DEFAULTS]
 listen_ip_addr = {{ contrail_haproxy_address }}
 listen_port = 5998
-zk_server_ip = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}{% if not loop.last %}, {% endif %}{% endfor %}
+zk_server_ip = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}{% if not loop.last %}, {% endif %}{% endfor %}
 
 zk_server_port = 2181
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 log_file = /var/log/contrail/contrail-discovery.log
 log_level = SYS_NOTICE
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-query-engine-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-query-engine-conf.j2
index 7564d26d..3eb20443 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-query-engine-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-query-engine-conf.j2
@@ -1,7 +1,7 @@
 [DEFAULT]
 hostip = {{ contrail_address }}
-#cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ ip_settings[cur_host]['mgmt']['ip'] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+#cassandra_server_list = {% for cur_host in groups['controller'] %}{{ ip_settings[cur_host]['mgmt']['ip'] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 collectors = {{ contrail_haproxy_address }}:8086
 http_server_port = 8091
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-schema-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-schema-conf.j2
index 46995f75..f65ab129 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-schema-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-schema-conf.j2
@@ -7,9 +7,9 @@ ifmap_password = schema-transformer
 #api_server_ip = {{ ip_settings[haproxy_hosts.keys()[0]]['mgmt']['ip'] }}
 api_server_ip = {{ contrail_haproxy_address }}
 api_server_port = 8082
-zk_server_ip = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host]  }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
+zk_server_ip = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host]  }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
 
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 disc_server_ip = {{ contrail_haproxy_address }}
 disc_server_port = 5998
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-svc-monitor-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-svc-monitor-conf.j2
index 51740de9..0f612360 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-svc-monitor-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-svc-monitor-conf.j2
@@ -5,9 +5,9 @@ ifmap_username = svc-monitor
 ifmap_password = svc-monitor
 api_server_ip = {{ contrail_haproxy_address }}
 api_server_port = 8082
-zk_server_ip = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
+zk_server_ip = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:2181{% if not loop.last %}, {% endif %}{% endfor %}
 
-cassandra_server_list = {% for cur_host in groups['opencontrail_database'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
+cassandra_server_list = {% for cur_host in groups['controller'] %}{{ haproxy_hosts[cur_host] }}:9160{% if not loop.last %} {% endif %}{% endfor %}
 
 disc_server_ip = {{ contrail_haproxy_address }}
 disc_server_port = 5998
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-vrouter-agent-conf.j2 b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-vrouter-agent-conf.j2
index d0d6b966..61a517c9 100755
--- a/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-vrouter-agent-conf.j2
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/provision/contrail-vrouter-agent-conf.j2
@@ -66,7 +66,7 @@ platform = default
 physical_interface_address =
 
 # MAC address of device used by dpdk
-physical_interface_mac = {{ hostvars[inventory_hostname][contrail_vhost_device]['macaddress'] }}
+physical_interface_mac = {{ hostvars[inventory_hostname]['ansible_'+contrail_vhost_device]['macaddress'] }}
 
 [DISCOVERY]
 # If COLLECTOR and/or CONTROL-NODE and/or DNS is not specified this section is
@@ -77,7 +77,7 @@ server = {{ contrail_haproxy_address }}
 
 # Number of control-nodes info to be provided by Discovery service. Possible
 # values are 1 and 2
-max_control_nodes = {{ groups['opencontrail_control'] | length }}
+max_control_nodes = {{ groups['controller'] | length }}
 
 [DNS]
 # IP address and port to be used to connect to dns-node. Maximum of 2 IP
@@ -127,7 +127,7 @@ control_network_ip = {{ contrail_haproxy_address }}
 name = vhost0
 
 # IP address and prefix in ip/prefix_len format
-ip = {{ contraili_vhost_address }}/{{ contrail_prefixlen }}
+ip = {{ contrail_vhost_address }}/{{ contrail_prefixlen }}
 
 # Gateway IP address for virtual host
 gateway = {{ contrail_vhost_gateway }}
diff --git a/deploy/adapters/ansible/roles/open-contrail/templates/vrouter-functions.sh b/deploy/adapters/ansible/roles/open-contrail/templates/vrouter-functions.sh
new file mode 100755
index 00000000..a6208660
--- /dev/null
+++ b/deploy/adapters/ansible/roles/open-contrail/templates/vrouter-functions.sh
@@ -0,0 +1,215 @@
+#!/bin/bash
+
+source /etc/contrail/agent_param
+
+function pkt_setup () {
+    for f in /sys/class/net/$1/queues/rx-*
+    do
+        q="$(echo $f | cut -d '-' -f2)"
+        r=$(($q%32))
+        s=$(($q/32))
+        ((mask=1<<$r))
+        str=(`printf "%x" $mask`)
+        if [ $s -gt 0 ]; then
+            for ((i=0; i < $s; i++))
+            do
+                str+=,00000000
+            done
+        fi
+        echo $str > $f/rps_cpus
+    done
+}
+
+function insert_vrouter() {
+    if cat $CONFIG | grep '^\s*platform\s*=\s*dpdk\b' &>/dev/null; then
+        vrouter_dpdk_start
+        return $?
+    fi
+
+    grep $kmod /proc/modules 1>/dev/null 2>&1
+    if [ $? != 0 ]; then 
+        insmod /var/lib/dkms/vrouter/2.21/build/vrouter.ko
+        if [ $? != 0 ]
+        then
+            echo "$(date) : Error inserting vrouter module"
+            return 1
+        fi
+
+        if [ -f /sys/class/net/pkt1/queues/rx-0/rps_cpus ]; then
+            pkt_setup pkt1
+        fi
+        if [ -f /sys/class/net/pkt2/queues/rx-0/rps_cpus ]; then
+            pkt_setup pkt2
+        fi
+        if [ -f /sys/class/net/pkt3/queues/rx-0/rps_cpus ]; then
+            pkt_setup pkt3
+        fi
+    fi
+
+    # check if vhost0 is not present, then create vhost0 and $dev
+    if [ ! -L /sys/class/net/vhost0 ]; then
+        echo "$(date): Creating vhost interface: $DEVICE."
+        # for bonding interfaces
+        loops=0
+        while [ ! -f /sys/class/net/$dev/address ]
+        do
+            sleep 1
+            loops=$(($loops + 1))
+            if [ $loops -ge 60 ]; then
+                echo "Unable to look at /sys/class/net/$dev/address"
+                return 1
+            fi
+        done
+
+        DEV_MAC=$(cat /sys/class/net/$dev/address)
+        vif --create $DEVICE --mac $DEV_MAC
+        if [ $? != 0 ]; then
+            echo "$(date): Error creating interface: $DEVICE"
+        fi
+
+
+        echo "$(date): Adding $dev to vrouter"
+        DEV_MAC=$(cat /sys/class/net/$dev/address)
+        vif --add $dev --mac $DEV_MAC --vrf 0 --vhost-phys --type physical
+        if [ $? != 0 ]; then
+            echo "$(date): Error adding $dev to vrouter"
+        fi
+
+        vif --add $DEVICE --mac $DEV_MAC --vrf 0 --type vhost --xconnect $dev
+        if [ $? != 0 ]; then
+            echo "$(date): Error adding $DEVICE to vrouter"
+        fi
+    fi
+    return 0
+}
+
+function vrouter_dpdk_start() {
+    # wait for vRouter/DPDK to start
+    echo "$(date): Waiting for vRouter/DPDK to start..."
+    service ${VROUTER_SERVICE} start
+    loops=0
+    while ! is_vrouter_dpdk_running
+    do
+        sleep 1
+        loops=$(($loops + 1))
+        if [ $loops -ge 60 ]; then
+            echo "No vRouter/DPDK running."
+            echo "Please check if ${VROUTER_SERVICE} service is up and running."
+            return 1
+        fi
+    done
+
+    # TODO: at the moment we have no interface deletion, so this loop might
+    #       be unnecessary in the future
+    echo "$(date): Waiting for Agent to configure $DEVICE..."
+    loops=0
+    while [ ! -L /sys/class/net/vhost0 ]
+    do
+        sleep 1
+        loops=$(($loops + 1))
+        if [ $loops -ge 10 ]; then
+            break
+        fi
+    done
+
+    # check if vhost0 is not present, then create vhost0 and $dev
+    if [ ! -L /sys/class/net/vhost0 ]; then
+        echo "$(date): Creating vhost interface: $DEVICE."
+        agent_conf_read
+
+        DEV_MAC=${physical_interface_mac}
+        DEV_PCI=${physical_interface_address}
+
+        if [ -z "${DEV_MAC}" -o -z "${DEV_PCI}" ]; then
+            echo "No device configuration found in ${CONFIG}"
+            return 1
+        fi
+
+        # TODO: the vhost creation is happening later in vif --add
+#        vif --create $DEVICE --mac $DEV_MAC
+#        if [ $? != 0 ]; then
+#            echo "$(date): Error creating interface: $DEVICE"
+#        fi
+
+        echo "$(date): Adding $dev to vrouter"
+        # add DPDK ethdev 0 as a physical interface
+        vif --add 0 --mac $DEV_MAC --vrf 0 --vhost-phys --type physical --pmd --id 0
+        if [ $? != 0 ]; then
+            echo "$(date): Error adding $dev to vrouter"
+        fi
+
+        # TODO: vif --xconnect seems does not work without --id parameter?
+        vif --add $DEVICE --mac $DEV_MAC --vrf 0 --type vhost --xconnect 0 --pmd --id 1
+        if [ $? != 0 ]; then
+            echo "$(date): Error adding $DEVICE to vrouter"
+        fi
+    fi
+    return 0
+}
+
+DPDK_BIND=/opt/contrail/bin/dpdk_nic_bind.py
+VROUTER_SERVICE="supervisor-vrouter"
+
+function is_vrouter_dpdk_running() {
+    # check for NetLink TCP socket
+    lsof -ni:20914 -sTCP:LISTEN > /dev/null
+
+    return $?
+}
+
+function agent_conf_read() {
+    eval `cat ${CONFIG} | grep -E '^\s*physical_\w+\s*='`
+}
+
+function vrouter_dpdk_if_bind() {
+    if [ ! -s /sys/class/net/${dev}/address ]; then
+        echo "No ${dev} device found."
+        ${DPDK_BIND} --status
+        return 1
+    fi
+
+    modprobe igb_uio
+    # multiple kthreads for port monitoring
+    modprobe rte_kni kthread_mode=multiple
+
+    ${DPDK_BIND} --force --bind=igb_uio $dev
+    ${DPDK_BIND} --status
+}
+
+function vrouter_dpdk_if_unbind() {
+    if [ -s /sys/class/net/${dev}/address ]; then
+        echo "Device ${dev} is already unbinded."
+        ${DPDK_BIND} --status
+        return 1
+    fi
+
+    agent_conf_read
+
+    DEV_PCI=${physical_interface_address}
+    DEV_DRIVER=`lspci -vmmks ${DEV_PCI} | grep 'Module:' | cut -d $'\t' -f 2`
+
+    if [ -z "${DEV_DRIVER}" -o -z "${DEV_PCI}" ]; then
+        echo "No device ${dev} configuration found in ${AGENT_DPDK_PARAMS_FILE}"
+        return 1
+    fi
+
+    # wait for vRouter/DPDK to stop
+    echo "$(date): Waiting for vRouter/DPDK to stop..."
+    loops=0
+    while is_vrouter_dpdk_running
+    do
+        sleep 1
+        loops=$(($loops + 1))
+        if [ $loops -ge 60 ]; then
+            echo "vRouter/DPDK is still running."
+            echo "Please try to stop ${VROUTER_SERVICE} service."
+            return 1
+        fi
+    done
+
+    ${DPDK_BIND} --force --bind=${DEV_DRIVER} ${DEV_PCI}
+    ${DPDK_BIND} --status
+
+    rmmod rte_kni
+    rmmod igb_uio
+}