Fix functions for fuel installer

JIRA: DOCTOR-58

Change-Id: Ic9326791007a3afa754cc64423b46adc17c59dc0
Signed-off-by: Ryota MIBU <r-mibu@cq.jp.nec.com>

4 files changed, 120 insertions, 87 deletions

diff --git a/tests/functions-common b/tests/functions-common
index 08d489e5..7928f20c 100644
--- a/tests/functions-common
+++ b/tests/functions-common
@@ -4,7 +4,7 @@
 # is_set env-var
 function is_set {
     local var=\$"$1"
-    eval "[ -n \"$var\" ]"
+    eval "[[ -n \"$var\" ]]"
 }
 
 # Prints backtrace info
@@ -41,6 +41,8 @@ function err {
 # die $LINENO "message"
 function die {
     local exitcode=$?
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
     set +o xtrace
     local line=$1; shift
     if [ $exitcode == 0 ]; then
@@ -50,6 +52,7 @@ function die {
     err $line "$*"
     # Give buffers a second to flush
     sleep 1
+    $xtrace
     exit $exitcode
 }
 
diff --git a/tests/lib/inspectors/sample b/tests/lib/inspectors/sample
index cd21a008..d09f7ad0 100644
--- a/tests/lib/inspectors/sample
+++ b/tests/lib/inspectors/sample
@@ -2,6 +2,8 @@
 
 function start_inspector_sample {
     pgrep -f "python inspector.py" && return 0
+    # TODO (r-mibu): make sample inspector use keystone v3 api
+    OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \
     python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
 }
 
diff --git a/tests/lib/installers/fuel b/tests/lib/installers/fuel
index 34a86922..31fe1fb6 100644
--- a/tests/lib/installers/fuel
+++ b/tests/lib/installers/fuel
@@ -1,107 +1,133 @@
 #!/bin/bash
 
+if [[ "congress " == "$INSPECTOR_TYPE" ]]; then
+    die $LINENO "fuel does not support congress yet..."
+fi
+
 function get_installer_ip {
-    local instack_mac=$(sudo virsh domiflist fuel-opnfv | awk '/pxebr/{print $5}')
+    ssh_opts_cpu="$ssh_opts -i instack_key"
+    is_set INSTALLER_IP && return
+    local instack_mac=$(sudo virsh domiflist fuel-master | awk '/fuel1/{print $5}')
     INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk '{print $1}')
     die_if_not_set $LINENO $INSTALLER_IP "No installer IP"
 }
 
+function get_controller_ips {
+    is_set CONTROLLER_IPS && return
+    get_installer_ip
+    CONTROLLER_IPS=$(ssh $ssh_opts_cpu root@$INSTALLER_IP \
+                     "fuel node | grep controller | cut -d '|' -f 5|xargs")
+    die_if_not_set $LINENO $CONTROLLER_IPS "No controller IPs"
+}
+
 function installer_get_ssh_keys {
+    if [[ -e instack_key ]]; then
+        echo "test existing instack_key..."
+        ssh $ssh_opts_cpu root@${INSTALLER_IP} "hostname" && return
+    fi
+    echo "getting instack_key from fuel node..."
     sshpass -p r00tme scp $ssh_opts root@${INSTALLER_IP}:.ssh/id_rsa instack_key
     sudo chown $(whoami):$(whoami) instack_key
     chmod 400 instack_key
-    ssh_opts_cpu+=" -i instack_key"
 }
 
 function installer_apply_patches {
-    cat > set_conf.sh << 'END_TXT'
-#!/bin/bash
-if [ -e /etc/ceilometer/event_pipeline.yaml ]; then
-    if ! grep -q '^ *- notifier://?topic=alarm.all$' /etc/ceilometer/event_pipeline.yaml; then
-        sed -i 's|- notifier://|- notifier://?topic=alarm.all|' /etc/ceilometer/event_pipeline.yaml
-        echo "modify the ceilometer config"
-        service ceilometer-agent-notification restart
+    if ! openstack flavor show $VM_FLAVOR ; then
+        openstack flavor create --ram 512 --disk 1 $VM_FLAVOR \
+            && touch created_doctor_flavor
     fi
-else
-    echo "ceilometer event_pipeline.yaml file does not exist"
-    exit 1
-fi
-if [ -e /etc/nova/nova.conf ]; then
-    if ! grep -q '^notification_driver=messaging$' /etc/nova/nova.conf; then
-        sed -i -r 's/notification_driver=/notification_driver=messaging/g' /etc/nova/nova.conf
-        echo "modify nova config"
-        service nova-api restart
-    fi
-else
-    echo "nova.conf file does not exist"
-    exit 1
-fi
-exit 0
-END_TXT
-
-    chmod +x set_conf.sh
-    CONTROLLER_IP=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \
-         "fuel node | grep controller | cut -d '|' -f 5|xargs")
-    for node in $CONTROLLER_IP;do
-        scp $ssh_opts_cpu set_conf.sh "root@$node:"
-        ssh $ssh_opts_cpu "root@$node" './set_conf.sh > set_conf.log 2>&1 &'
-        sleep 1
-        scp $ssh_opts_cpu "root@$node:set_conf.log" set_conf_$node.log
-    done
 
-    if grep -q "modify the ceilometer config" set_conf_*.log ; then
-        NEED_TO_RESTORE_CEILOMETER=true
-    fi
-    if grep -q "modify nova config" set_conf_*.log ; then
-        NEED_TO_RESTORE_NOVA=true
-    fi
+    # TODO(r-mibu): fix the followings in upstream (fuel)
+    get_controller_ips
+    for node in $CONTROLLER_IPS;do
+        echo "check controller configuration for doctor ($node)"
+        ssh $ssh_opts_cpu "root@$node" '
+            set -x
+            date
+            echo "### apply patches (installer=fuel)"
+
+            ep_conf=/etc/ceilometer/event_pipeline.yaml
+            entry="- notifier://?topic=alarm.all"
+            if ! grep -q -e "$entry" $ep_conf; then
+                echo "modify the ceilometer config"
+                echo "          $entry    # added by doctor script" >> $ep_conf
+                service ceilometer-agent-notification restart
+            fi
 
-    echo "waiting service restart..."
-    sleep 60
+            # TODO(r-mibu): enable this section once congress 4.0.0 is available
+            if false; then
+            co_conf=/etc/congress/congress.conf
+            entry="congress.datasources.doctor_driver.DoctorDriver"
+            if ! grep -q -e "^drivers.*$entry" $co_conf; then
+                echo "modify the congress config"
+                sed -i -e "/^drivers/s/$/,$entry    # added by doctor script/" \
+                    $co_conf
+                service congress-server restart
+            fi
 
+            rule="-m multiport -p tcp --dports 1789"
+            rule+=" -m comment --comment doctor-congress"
+            rule+=" -j ACCEPT"
+            if ! iptables -C INPUT $rule; then
+                iptables -I INPUT $rule
+            fi
+
+            ha_conf=/etc/haproxy/conf.d/180-congress.cfg
+            if [[ ! -e $ha_conf ]]; then
+                sed -e "1i# generated by doctor script" \
+                    -e "s/9696/1789/" \
+                    -e "s/neutron/congress/" \
+                    /etc/haproxy/conf.d/085-neutron.cfg > $ha_conf
+                ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart
+            fi
+            fi
+            ' > installer_apply_patches_$node.log 2>&1
+    done
 }
 
 function cleanup_installer_fuel {
-   if ! ($NEED_TO_RESTORE_CEILOMETER || $NEED_TO_RESTORE_NOVA) ; then
-       echo "Don't need to restore config"
-       exit 0
-   fi
-
-   echo "restore the configuration..."
-   cat > restore_conf.sh << 'END_TXT'
-#!/bin/bash
-if @NEED_TO_RESTORE_CEILOMETER@ ; then
-    if [ -e /etc/ceilometer/event_pipeline.yaml ]; then
-        if grep -q '^ *- notifier://?topic=alarm.all$' /etc/ceilometer/event_pipeline.yaml; then
-            sed -i 's|- notifier://?topic=alarm.all|- notifier://|' /etc/ceilometer/event_pipeline.yaml
-            service ceilometer-agent-notification restart
-        fi
-    else
-        echo "ceilometer event_pipeline.yaml file does not exist"
-        exit 1
+    if [[ -e created_doctor_flavor ]] && openstack flavor show $VM_FLAVOR ; then
+        openstack flavor delete $VM_FLAVOR && rm -f created_doctor_flavor
     fi
-fi
-if @NEED_TO_RESTORE_NOVA@ ; then
-    if [ -e /etc/nova/nova.conf ]; then
-        if grep -q '^notification_driver=messaging$' /etc/nova/nova.conf; then
-            sed -i -r 's/notification_driver=messaging/notification_driver=/g' /etc/nova/nova.conf
-            service nova-api restart
-        fi
-    else
-        echo "nova.conf file does not exist"
-        exit 1
-    fi
-fi
-exit 0
-END_TXT
-   sed -i -e "s/@NEED_TO_RESTORE_CEILOMETER@/$NEED_TO_RESTORE_CEILOMETER/" restore_conf.sh
-   sed -i -e "s/@NEED_TO_RESTORE_NOVA@/$NEED_TO_RESTORE_NOVA/" restore_conf.sh
-   chmod +x restore_conf.sh
-   for node in $CONTROLLER_IP;do
-       scp $ssh_opts_cpu restore_conf.sh "root@$node:"
-       ssh $ssh_opts_cpu "root@$node" './restore_conf.sh > restore_conf.log 2>&1 &'
-   done
-
-   echo "waiting service restart..."
-   sleep 60
+
+    # TODO(r-mibu): fix the followings in upstream (fuel)
+    get_controller_ips
+    for node in $CONTROLLER_IPS;do
+        echo "restore controller configuration if touched ($node)"
+        ssh $ssh_opts_cpu "root@$node" '
+            set -x
+            echo "### revert patches (installer=fuel)"
+            date
+
+            # TODO(r-mibu): enable this section once congress 4.0.0 is available
+            if false; then
+            ha_conf=/etc/haproxy/conf.d/180-congress.cfg
+            if grep -q "# generated by doctor script" $ha_conf; then
+                rm -f $ha_conf
+                ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart
+            fi
+
+            rule="-m multiport -p tcp --dports 1789"
+            rule+=" -m comment --comment doctor-congress"
+            rule+=" -j ACCEPT"
+            if iptables -C INPUT $rule; then
+                iptables -D INPUT $rule
+            fi
+
+            co_conf=/etc/congress/congress.conf
+            entry="congress.datasources.doctor_driver.DoctorDriver"
+            if grep -q -e "^drivers.*$entry    # added by doctor script" $co_conf; then
+                echo "modify the congress config"
+                sed -i -e "/^drivers/s/^\(.*\),$entry/\1/" $co_conf
+                service congress-server restart
+            fi
+            fi
+
+            ep_conf=/etc/ceilometer/event_pipeline.yaml
+            if grep -q "# added by doctor script" $ep_conf; then
+                sed -ie "/# added by doctor script/d" $ep_conf
+                service ceilometer-agent-notification restart
+            fi
+            ' >> installer_apply_patches_$node.log 2>&1
+    done
 }
diff --git a/tests/run.sh b/tests/run.sh
index e7afb518..227c6ea1 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -179,8 +179,7 @@ start_consumer() {
                              nova list | grep ' overcloud-controller-[0-9] ' \
                              | sed -e 's/^.*ctlplane=//' -e 's/ *|\$//'")
         elif is_installer fuel; then
-            CONTROLLER_IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \
-                            "fuel node | grep controller | cut -d '|' -f 5|xargs")
+            get_controller_ips
         fi
 
         die_if_not_set $LINENO CONTROLLER_IPS "Could not get CONTROLLER_IPS."
@@ -236,6 +235,7 @@ inject_failure() {
     cat > disable_network.sh << 'END_TXT'
 #!/bin/bash -x
 dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $7}')
+[[ -n "$dev" ]] || dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $5}')
 sleep 1
 sudo ip link set $dev down
 echo "doctor set host down at" $(date "+%s.%N")
@@ -309,6 +309,8 @@ unset_forced_down_hosts() {
     for host in $(openstack compute service list --service nova-compute \
                   -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
     do
+        # TODO (r-mibu): make sample inspector use keystone v3 api
+        OS_AUTH_URL=${OS_AUTH_URL/v3/v2.0} \
         python ./nova_force_down.py $host --unset
     done