Merge "HA testcase improvement" into stable/danube

11 files changed, 83 insertions, 25 deletions

diff --git a/tests/ci/clean_images.sh b/tests/ci/clean_images.sh
index 27da9e279..cb5410ab2 100755
--- a/tests/ci/clean_images.sh
+++ b/tests/ci/clean_images.sh
@@ -15,18 +15,24 @@ cleanup()
     echo
     echo "========== Cleanup =========="
 
-    if ! openstack image list; then
+    if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+        SECURE="--insecure"
+    else
+        SECURE=""
+    fi
+
+    if ! openstack "${SECURE}" image list; then
         return
     fi
 
-    for image in $(openstack image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \
+    for image in $(openstack "${SECURE}" image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \
         | awk '{print $2}'); do
         echo "Deleting image $image..."
-        openstack image delete $image || true
+        openstack "${SECURE}" image delete $image || true
     done
 
-    openstack flavor delete yardstick-flavor &> /dev/null || true
-    openstack flavor delete storperf &> /dev/null || true
+    openstack "${SECURE}" flavor delete yardstick-flavor &> /dev/null || true
+    openstack "${SECURE}" flavor delete storperf &> /dev/null || true
 }
 
 main()
diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh
index 0ebe9b77c..a5114f419 100755
--- a/tests/ci/load_images.sh
+++ b/tests/ci/load_images.sh
@@ -88,7 +88,7 @@ load_yardstick_image()
         if [ ! -f "${CLOUD_KERNEL}" ]; then
             tar xf "${CLOUD_IMAGE}" "${CLOUD_KERNEL##**/}"
         fi
-        create_kernel=$(openstack image create \
+        create_kernel=$(openstack "${SECURE}" image create \
                 --public \
                 --disk-format qcow2 \
                 --container-format bare \
@@ -119,7 +119,7 @@ load_yardstick_image()
     fi
 
     if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then
-        output=$(eval openstack image create \
+        output=$(eval openstack "${SECURE}" image create \
             --public \
             --disk-format raw \
             --container-format bare \
@@ -127,7 +127,7 @@ load_yardstick_image()
             --file ${RAW_IMAGE} \
             yardstick-image)
     else
-        output=$(eval openstack image create \
+        output=$(eval openstack "${SECURE}" image create \
             --public \
             --disk-format qcow2 \
             --container-format bare \
@@ -156,7 +156,7 @@ load_yardstick_image()
 
 load_cirros_image()
 {
-    if [[ -n $(openstack image list | grep -e Cirros-0.3.5) ]]; then
+    if [[ -n $(openstack "${SECURE}" image list | grep -e Cirros-0.3.5) ]]; then
         echo "Cirros-0.3.5 image already exist, skip loading cirros image"
     else
         echo
@@ -170,7 +170,7 @@ load_cirros_image()
             EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
         fi
 
-        output=$(openstack image create \
+        output=$(openstack "${SECURE}" image create \
             --disk-format qcow2 \
             --container-format bare \
             ${EXTRA_PARAMS} \
@@ -201,7 +201,7 @@ load_ubuntu_image()
         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
     fi
 
-    output=$(openstack image create \
+    output=$(openstack "${SECURE}" image create \
         --disk-format qcow2 \
         --container-format bare \
         $EXTRA_PARAMS \
@@ -221,26 +221,26 @@ load_ubuntu_image()
 
 create_nova_flavor()
 {
-    if ! openstack flavor list | grep -q yardstick-flavor; then
+    if ! openstack "${SECURE}" flavor list | grep -q yardstick-flavor; then
         echo
         echo "========== Creating yardstick-flavor =========="
         # Create the nova flavor used by some sample test cases
-        openstack flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor
+        openstack "${SECURE}" flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor
         # DPDK-enabled OVS requires guest memory to be backed by large pages
         if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then
-            openstack flavor set --property hw:mem_page_size=large yardstick-flavor
+            openstack "${SECURE}" flavor set --property hw:mem_page_size=large yardstick-flavor
         fi
         # VPP requires guest memory to be backed by large pages
         if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
-            openstack flavor set --property hw:mem_page_size=large yardstick-flavor
+            openstack "${SECURE}" flavor set --property hw:mem_page_size=large yardstick-flavor
         fi
     fi
 
-    if ! openstack flavor list | grep -q storperf; then
+    if ! openstack "${SECURE}" flavor list | grep -q storperf; then
         echo
         echo "========== Creating storperf flavor =========="
         # Create the nova flavor used by storperf test case
-        openstack flavor create --id auto --ram 8192 --disk 4 --vcpus 2 storperf
+        openstack "${SECURE}" flavor create --id auto --ram 8192 --disk 4 --vcpus 2 storperf
     fi
 }
 
@@ -256,6 +256,12 @@ main()
         RAW_IMAGE='/home/opnfv/images/yardstick-image.tar.gz'
     fi
 
+    if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+        SECURE="--insecure"
+    else
+        SECURE=""
+    fi
+
     build_yardstick_image
     load_yardstick_image
     if [ "${YARD_IMG_ARCH}" == "arm64" ]; then
diff --git a/tests/unit/benchmark/scenarios/availability/test_scenario_general.py b/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
index ea54fbb9b..de2170b16 100644
--- a/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
+++ b/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
@@ -56,6 +56,7 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         mock_obj = mock.Mock()
         mock_obj.createActionPlayer.side_effect = KeyError('Wrong')
         ins.director = mock_obj
+        ins.director.data = {}
         ins.run({})
         ins.teardown()
 
@@ -64,5 +65,6 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         mock_obj = mock.Mock()
         mock_obj.verify.return_value = False
         ins.director = mock_obj
+        ins.director.data = {}
         ins.run({})
         ins.teardown()
diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
index e0e6cf3bf..f7ab23dcd 100644
--- a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
@@ -66,3 +66,5 @@ class ProcessAttacker(BaseAttacker):
             exit_status, stdout, stderr = self.connection.execute(
                 "sudo /bin/bash -s {0} ".format(self.service_name),
                 stdin=stdin_file)
+        if exit_status:
+            LOG.info("Fail to restart service!")
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
index a6a3e96ca..a865b6551 100755
--- a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
@@ -16,10 +16,13 @@ set -e
 process_name=$1
 
 if [ "$process_name" = "keystone" ]; then
-    killall -9 -u $process_name
+    for pid in $(ps aux | grep "keystone" | grep -iv heartbeat | grep -iv monitor | grep -v grep | grep -v /bin/sh | awk '{print $2}'); \
+        do
+            kill -9 "${pid}"
+        done
 else
-    for pid in `ps aux | grep "/usr/.*/${process_name}" | grep -v grep | grep -v /bin/sh | awk '{print $2}'`; \
+    for pid in $(pgrep -f "/usr/.*/${process_name}");
         do
-            kill -9 ${pid}
+            kill -9 "${pid}"
         done
 fi
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
index 941563e7c..8737836e2 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
index e998464c7..617dcf8a3 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor delete $1
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor delete $1
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
index 1b0739602..9b413c965 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
@@ -13,4 +13,10 @@
 
 set -e
 
-nova flavor-list
\ No newline at end of file
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor list
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
index 033a2d721..d757bd88d 100644
--- a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 from __future__ import absolute_import
+
+import os
 import logging
 import subprocess
 import traceback
@@ -53,6 +55,14 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor):
 
         self.cmd = self._config["command_name"]
 
+        try:
+            cacert = os.environ['OS_CACERT']
+        except KeyError:
+            pass
+        else:
+            if cacert.lower() == "false":
+                self.cmd = self.cmd + " --insecure"
+
     def monitor_func(self):
         exit_status = 0
         exit_status, stdout = _execute_shell_command(self.cmd)
diff --git a/yardstick/benchmark/scenarios/availability/scenario_general.py b/yardstick/benchmark/scenarios/availability/scenario_general.py
index 689d33a34..28bec8aff 100644
--- a/yardstick/benchmark/scenarios/availability/scenario_general.py
+++ b/yardstick/benchmark/scenarios/availability/scenario_general.py
@@ -54,7 +54,18 @@ class ScenarioGeneral(base.Scenario):
                 pass
 
         self.director.stopMonitors()
-        if self.director.verify():
+
+        verify_result = self.director.verify()
+
+        for k, v in self.director.data.items():
+            if v == 0:
+                result['sla_pass'] = 0
+                verify_result = False
+                LOG.info(
+                    "\033[92m The service process not found in the host \
+envrioment, the HA test case NOT pass")
+
+        if verify_result:
             result['sla_pass'] = 1
             LOG.info(
                 "\033[92m Congratulations, "
diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
index 2e829714d..2f0012ecf 100755
--- a/yardstick/benchmark/scenarios/availability/serviceha.py
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -71,7 +71,7 @@ class ServiceHA(base.Scenario):
 
         sla_pass = self.monitorMgr.verify_SLA()
         for k, v in self.data.items():
-            if self.data[k] == 0:
+            if v == 0:
                 result['sla_pass'] = 0
                 LOG.info("The service process not found in the host envrioment, \
 the HA test case NOT pass")