HA testcase improvement

This patch improve HA test case in the following aspects:
1. the "GeneralHA" type now will check if the target service process in the controller node.
2. support ignore server certificate
3. add debug log for recovering service failed
4. improve method to kill keystone process

Change-Id: I9ae7ab54391fe41d5d7f3e4951a7ac2e3ba75968
Signed-off-by: JingLu5 <lvjing5@huawei.com>
(cherry picked from commit 5b99f1532ec4d15258ec86e970acd2904954b3bc)

8 files changed, 52 insertions, 8 deletions

diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
index e0e6cf3bf..f7ab23dcd 100644
--- a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
@@ -66,3 +66,5 @@ class ProcessAttacker(BaseAttacker):
             exit_status, stdout, stderr = self.connection.execute(
                 "sudo /bin/bash -s {0} ".format(self.service_name),
                 stdin=stdin_file)
+        if exit_status:
+            LOG.info("Fail to restart service!")
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
index a6a3e96ca..a865b6551 100755
--- a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
@@ -16,10 +16,13 @@ set -e
 process_name=$1
 
 if [ "$process_name" = "keystone" ]; then
-    killall -9 -u $process_name
+    for pid in $(ps aux | grep "keystone" | grep -iv heartbeat | grep -iv monitor | grep -v grep | grep -v /bin/sh | awk '{print $2}'); \
+        do
+            kill -9 "${pid}"
+        done
 else
-    for pid in `ps aux | grep "/usr/.*/${process_name}" | grep -v grep | grep -v /bin/sh | awk '{print $2}'`; \
+    for pid in $(pgrep -f "/usr/.*/${process_name}");
         do
-            kill -9 ${pid}
+            kill -9 "${pid}"
         done
 fi
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
index 941563e7c..8737836e2 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
index e998464c7..617dcf8a3 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor delete $1
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor delete $1
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
index 1b0739602..9b413c965 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
@@ -13,4 +13,10 @@
 
 set -e
 
-nova flavor-list
\ No newline at end of file
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor list
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
index 033a2d721..d757bd88d 100644
--- a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 from __future__ import absolute_import
+
+import os
 import logging
 import subprocess
 import traceback
@@ -53,6 +55,14 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor):
 
         self.cmd = self._config["command_name"]
 
+        try:
+            cacert = os.environ['OS_CACERT']
+        except KeyError:
+            pass
+        else:
+            if cacert.lower() == "false":
+                self.cmd = self.cmd + " --insecure"
+
     def monitor_func(self):
         exit_status = 0
         exit_status, stdout = _execute_shell_command(self.cmd)
diff --git a/yardstick/benchmark/scenarios/availability/scenario_general.py b/yardstick/benchmark/scenarios/availability/scenario_general.py
index 689d33a34..28bec8aff 100644
--- a/yardstick/benchmark/scenarios/availability/scenario_general.py
+++ b/yardstick/benchmark/scenarios/availability/scenario_general.py
@@ -54,7 +54,18 @@ class ScenarioGeneral(base.Scenario):
                 pass
 
         self.director.stopMonitors()
-        if self.director.verify():
+
+        verify_result = self.director.verify()
+
+        for k, v in self.director.data.items():
+            if v == 0:
+                result['sla_pass'] = 0
+                verify_result = False
+                LOG.info(
+                    "\033[92m The service process not found in the host \
+envrioment, the HA test case NOT pass")
+
+        if verify_result:
             result['sla_pass'] = 1
             LOG.info(
                 "\033[92m Congratulations, "
diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
index 2e829714d..2f0012ecf 100755
--- a/yardstick/benchmark/scenarios/availability/serviceha.py
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -71,7 +71,7 @@ class ServiceHA(base.Scenario):
 
         sla_pass = self.monitorMgr.verify_SLA()
         for k, v in self.data.items():
-            if self.data[k] == 0:
+            if v == 0:
                 result['sla_pass'] = 0
                 LOG.info("The service process not found in the host envrioment, \
 the HA test case NOT pass")