HA testcase improvement

This patch improve HA test case in the following aspects:
1. the "GeneralHA" type now will check if the target service process in the controller node.
2. support ignore server certificate
3. add debug log for recovering service failed
4. improve method to kill keystone process

Change-Id: I9ae7ab54391fe41d5d7f3e4951a7ac2e3ba75968
Signed-off-by: JingLu5 <lvjing5@huawei.com>
(cherry picked from commit 5b99f1532ec4d15258ec86e970acd2904954b3bc)

11 files changed, 83 insertions, 25 deletions

diff --git a/tests/ci/clean_images.sh b/tests/ci/clean_images.sh
index 27da9e279..cb5410ab2 100755
--- a/tests/ci/clean_images.sh
+++ b/tests/ci/clean_images.sh
@@ -15,18 +15,24 @@ cleanup()
     echo
     echo "========== Cleanup =========="
 
-    if ! openstack image list; then
+    if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+        SECURE="--insecure"
+    else
+        SECURE=""
+    fi
+
+    if ! openstack "${SECURE}" image list; then
         return
     fi
 
-    for image in $(openstack image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \
+    for image in $(openstack "${SECURE}" image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \
         | awk '{print $2}'); do
         echo "Deleting image $image..."
-        openstack image delete $image || true
+        openstack "${SECURE}" image delete $image || true
     done
 
-    openstack flavor delete yardstick-flavor &> /dev/null || true
-    openstack flavor delete storperf &> /dev/null || true
+    openstack "${SECURE}" flavor delete yardstick-flavor &> /dev/null || true
+    openstack "${SECURE}" flavor delete storperf &> /dev/null || true
 }
 
 main()
diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh
index 0ebe9b77c..a5114f419 100755
--- a/tests/ci/load_images.sh
+++ b/tests/ci/load_images.sh
@@ -88,7 +88,7 @@ load_yardstick_image()
         if [ ! -f "${CLOUD_KERNEL}" ]; then
             tar xf "${CLOUD_IMAGE}" "${CLOUD_KERNEL##**/}"
         fi
-        create_kernel=$(openstack image create \
+        create_kernel=$(openstack "${SECURE}" image create \
                 --public \
                 --disk-format qcow2 \
                 --container-format bare \
@@ -119,7 +119,7 @@ load_yardstick_image()
     fi
 
     if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then
-        output=$(eval openstack image create \
+        output=$(eval openstack "${SECURE}" image create \
             --public \
             --disk-format raw \
             --container-format bare \
@@ -127,7 +127,7 @@ load_yardstick_image()
             --file ${RAW_IMAGE} \
             yardstick-image)
     else
-        output=$(eval openstack image create \
+        output=$(eval openstack "${SECURE}" image create \
             --public \
             --disk-format qcow2 \
             --container-format bare \
@@ -156,7 +156,7 @@ load_yardstick_image()
 
 load_cirros_image()
 {
-    if [[ -n $(openstack image list | grep -e Cirros-0.3.5) ]]; then
+    if [[ -n $(openstack "${SECURE}" image list | grep -e Cirros-0.3.5) ]]; then
         echo "Cirros-0.3.5 image already exist, skip loading cirros image"
     else
         echo
@@ -170,7 +170,7 @@ load_cirros_image()
             EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
         fi
 
-        output=$(openstack image create \
+        output=$(openstack "${SECURE}" image create \
             --disk-format qcow2 \
             --container-format bare \
             ${EXTRA_PARAMS} \
@@ -201,7 +201,7 @@ load_ubuntu_image()
         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
     fi
 
-    output=$(openstack image create \
+    output=$(openstack "${SECURE}" image create \
         --disk-format qcow2 \
         --container-format bare \
         $EXTRA_PARAMS \
@@ -221,26 +221,26 @@ load_ubuntu_image()
 
 create_nova_flavor()
 {
-    if ! openstack flavor list | grep -q yardstick-flavor; then
+    if ! openstack "${SECURE}" flavor list | grep -q yardstick-flavor; then
         echo
         echo "========== Creating yardstick-flavor =========="
         # Create the nova flavor used by some sample test cases
-        openstack flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor
+        openstack "${SECURE}" flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor
         # DPDK-enabled OVS requires guest memory to be backed by large pages
         if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then
-            openstack flavor set --property hw:mem_page_size=large yardstick-flavor
+            openstack "${SECURE}" flavor set --property hw:mem_page_size=large yardstick-flavor
         fi
         # VPP requires guest memory to be backed by large pages
         if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
-            openstack flavor set --property hw:mem_page_size=large yardstick-flavor
+            openstack "${SECURE}" flavor set --property hw:mem_page_size=large yardstick-flavor
         fi
     fi
 
-    if ! openstack flavor list | grep -q storperf; then
+    if ! openstack "${SECURE}" flavor list | grep -q storperf; then
         echo
         echo "========== Creating storperf flavor =========="
         # Create the nova flavor used by storperf test case
-        openstack flavor create --id auto --ram 8192 --disk 4 --vcpus 2 storperf
+        openstack "${SECURE}" flavor create --id auto --ram 8192 --disk 4 --vcpus 2 storperf
     fi
 }
 
@@ -256,6 +256,12 @@ main()
         RAW_IMAGE='/home/opnfv/images/yardstick-image.tar.gz'
     fi
 
+    if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+        SECURE="--insecure"
+    else
+        SECURE=""
+    fi
+
     build_yardstick_image
     load_yardstick_image
     if [ "${YARD_IMG_ARCH}" == "arm64" ]; then
diff --git a/tests/unit/benchmark/scenarios/availability/test_scenario_general.py b/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
index ea54fbb9b..de2170b16 100644
--- a/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
+++ b/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
@@ -56,6 +56,7 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         mock_obj = mock.Mock()
         mock_obj.createActionPlayer.side_effect = KeyError('Wrong')
         ins.director = mock_obj
+        ins.director.data = {}
         ins.run({})
         ins.teardown()
 
@@ -64,5 +65,6 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         mock_obj = mock.Mock()
         mock_obj.verify.return_value = False
         ins.director = mock_obj
+        ins.director.data = {}
         ins.run({})
         ins.teardown()
diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
index e0e6cf3bf..f7ab23dcd 100644
--- a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
@@ -66,3 +66,5 @@ class ProcessAttacker(BaseAttacker):
             exit_status, stdout, stderr = self.connection.execute(
                 "sudo /bin/bash -s {0} ".format(self.service_name),
                 stdin=stdin_file)
+        if exit_status:
+            LOG.info("Fail to restart service!")
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
index a6a3e96ca..a865b6551 100755
--- a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
@@ -16,10 +16,13 @@ set -e
 process_name=$1
 
 if [ "$process_name" = "keystone" ]; then
-    killall -9 -u $process_name
+    for pid in $(ps aux | grep "keystone" | grep -iv heartbeat | grep -iv monitor | grep -v grep | grep -v /bin/sh | awk '{print $2}'); \
+        do
+            kill -9 "${pid}"
+        done
 else
-    for pid in `ps aux | grep "/usr/.*/${process_name}" | grep -v grep | grep -v /bin/sh | awk '{print $2}'`; \
+    for pid in $(pgrep -f "/usr/.*/${process_name}");
         do
-            kill -9 ${pid}
+            kill -9 "${pid}"
         done
 fi
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
index 941563e7c..8737836e2 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
index e998464c7..617dcf8a3 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor delete $1
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor delete $1
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
index 1b0739602..9b413c965 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
@@ -13,4 +13,10 @@
 
 set -e
 
-nova flavor-list
\ No newline at end of file
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor list
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
index 033a2d721..d757bd88d 100644
--- a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 from __future__ import absolute_import
+
+import os
 import logging
 import subprocess
 import traceback
@@ -53,6 +55,14 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor):
 
         self.cmd = self._config["command_name"]
 
+        try:
+            cacert = os.environ['OS_CACERT']
+        except KeyError:
+            pass
+        else:
+            if cacert.lower() == "false":
+                self.cmd = self.cmd + " --insecure"
+
     def monitor_func(self):
         exit_status = 0
         exit_status, stdout = _execute_shell_command(self.cmd)
diff --git a/yardstick/benchmark/scenarios/availability/scenario_general.py b/yardstick/benchmark/scenarios/availability/scenario_general.py
index 689d33a34..28bec8aff 100644
--- a/yardstick/benchmark/scenarios/availability/scenario_general.py
+++ b/yardstick/benchmark/scenarios/availability/scenario_general.py
@@ -54,7 +54,18 @@ class ScenarioGeneral(base.Scenario):
                 pass
 
         self.director.stopMonitors()
-        if self.director.verify():
+
+        verify_result = self.director.verify()
+
+        for k, v in self.director.data.items():
+            if v == 0:
+                result['sla_pass'] = 0
+                verify_result = False
+                LOG.info(
+                    "\033[92m The service process not found in the host \
+envrioment, the HA test case NOT pass")
+
+        if verify_result:
             result['sla_pass'] = 1
             LOG.info(
                 "\033[92m Congratulations, "
diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
index 2e829714d..2f0012ecf 100755
--- a/yardstick/benchmark/scenarios/availability/serviceha.py
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -71,7 +71,7 @@ class ServiceHA(base.Scenario):
 
         sla_pass = self.monitorMgr.verify_SLA()
         for k, v in self.data.items():
-            if self.data[k] == 0:
+            if v == 0:
                 result['sla_pass'] = 0
                 LOG.info("The service process not found in the host envrioment, \
 the HA test case NOT pass")