bugfix: Graceful shutdown of VM - improvement

Cleanup phase of PVVP scenario sometimes causes server reboot.
Following updates were made to prevent reboots:
* better generic process termination procedure
* ovsdb is terminated after vswitchd termination
* vswitchd is terminated directly instead of parent sudo process
* already running VNFs are terminated in case of failure during
VNF start()

Change-Id: Ic09d60d7bfdea01c84a2685ede3d0316f0d09be7
JIRA: VSPERF-271
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>

2 files changed, 24 insertions, 13 deletions

diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index c735062f..d108dc9a 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -21,6 +21,7 @@ import locale
 import re
 import subprocess
 import time
+import pexpect
 
 from conf import settings as S
 from conf import get_test_param
@@ -133,15 +134,24 @@ class IVnfQemu(IVnf):
         """
         Stops VNF instance gracefully first.
         """
-        # exit testpmd if needed
-        if self._guest_loopback == 'testpmd':
-            self.execute_and_wait('stop', 120, "Done")
-            self.execute_and_wait('quit', 120, "bye")
-
-        # turn off VM
-        self.execute_and_wait('poweroff', 120, "Power down")
-        # VM OS is off, but wait until qemu shutdowns
-        time.sleep(2)
+        try:
+            # exit testpmd if needed
+            if self._guest_loopback == 'testpmd':
+                self.execute_and_wait('stop', 120, "Done")
+                self.execute_and_wait('quit', 120, "bye")
+
+            # turn off VM
+            self.execute_and_wait('poweroff', 120, "Power down")
+
+        except pexpect.TIMEOUT:
+            self.kill()
+
+        # wait until qemu shutdowns
+        self._logger.debug('Wait for QEMU to terminate')
+        for dummy in range(30):
+            time.sleep(1)
+            if not self.is_running():
+                break
 
         # just for case that graceful shutdown failed
         super(IVnfQemu, self).stop()
diff --git a/vnfs/vnf/vnf.py b/vnfs/vnf/vnf.py
index 3dae2733..1410a0c4 100644
--- a/vnfs/vnf/vnf.py
+++ b/vnfs/vnf/vnf.py
@@ -51,11 +51,12 @@ class IVnf(tasks.Process):
         """
         Stops VNF instance.
         """
-        self._logger.info('Killing VNF...')
+        if self.is_running():
+            self._logger.info('Killing VNF...')
 
-        # force termination of VNF and wait for it to terminate; It will avoid
-        # sporadic reboot of host. (caused by hugepages or DPDK ports)
-        super(IVnf, self).kill(signal='-9', sleep=10)
+            # force termination of VNF and wait for it to terminate; It will avoid
+            # sporadic reboot of host. (caused by hugepages or DPDK ports)
+            super(IVnf, self).kill(signal='-9', sleep=10)
 
     def execute(self, cmd, delay=0):
         """