Test Case: OPNFV_YARDSTICK_TC023: VM availability during live migration

JIRA: YARDSTICK-174

This live migration test case is based on share storage, default share
storage is enabled.
This test case will do some config work. And do live migration and
calculate the migration time and downtime.

Change-Id: I6601601edebdd0ac6434ba632b1eba9e9bd4fda0
Signed-off-by: chenjiankun <chenjiankun1@huawei.com>

10 files changed, 599 insertions, 0 deletions

diff --git a/yardstick/benchmark/scenarios/base.py b/yardstick/benchmark/scenarios/base.py
index 5d3c36c38..3cb138dd8 100644
--- a/yardstick/benchmark/scenarios/base.py
+++ b/yardstick/benchmark/scenarios/base.py
@@ -63,3 +63,15 @@ class Scenario(object):
                 return scenario.__module__ + "." + scenario.__name__
 
         raise RuntimeError("No such scenario type %s" % scenario_type)
+
+    def _push_to_outputs(self, keys, values):
+        return dict(zip(keys, values))
+
+    def _change_obj_to_dict(self, obj):
+        dic = {}
+        for k, v in vars(obj).items():
+            try:
+                vars(v)
+            except TypeError:
+                dic[k] = v
+        return dic
diff --git a/yardstick/benchmark/scenarios/lib/__init__.py b/yardstick/benchmark/scenarios/lib/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/__init__.py
diff --git a/yardstick/benchmark/scenarios/lib/add_memory_load.py b/yardstick/benchmark/scenarios/lib/add_memory_load.py
new file mode 100644
index 000000000..26cf140d1
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/add_memory_load.py
@@ -0,0 +1,57 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class AddMemoryLoad(base.Scenario):
+    """Add memory load in server
+    """
+
+    __scenario_type__ = "AddMemoryLoad"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+
+        self.options = scenario_cfg.get('options', {})
+
+        self.client = ssh.SSH.from_node(self.context_cfg['host'])
+        self.client.wait(timeout=600)
+
+    def run(self, result):
+        self._add_load()
+
+    def _add_load(self):
+        try:
+            memory_load = self.options['memory_load']
+        except KeyError:
+            LOG.error('memory_load parameter must be provided')
+        else:
+            if float(memory_load) == 0:
+                return
+            cmd = 'free | awk "/Mem/ {print $2}"'
+            code, stdout, stderr = self.client.execute(cmd)
+            total = int(stdout.split()[1])
+            used = int(stdout.split()[2])
+            remain_memory = total * float(memory_load) - used
+            if remain_memory > 0:
+                count = remain_memory / 1024 / 128
+                LOG.info('Add %s vm load', count)
+                if count != 0:
+                    cmd = 'stress -t 10 -m {} --vm-keep'.format(count)
+                    self.client.execute(cmd)
diff --git a/yardstick/benchmark/scenarios/lib/check_numa_info.py b/yardstick/benchmark/scenarios/lib/check_numa_info.py
new file mode 100644
index 000000000..59a47547e
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/check_numa_info.py
@@ -0,0 +1,61 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class CheckNumaInfo(base.Scenario):
+    """
+    Execute a live migration for two hosts
+
+    """
+
+    __scenario_type__ = "CheckNumaInfo"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+
+        self.options = self.scenario_cfg.get('options', {})
+
+        self.cpu_set = self.options.get('cpu_set', '1,2,3,4,5,6')
+
+    def run(self, result):
+        info1 = self.options.get('info1')
+        info2 = self.options.get('info2')
+        LOG.debug('Origin numa info: %s', info1)
+        LOG.debug('Current numa info: %s', info2)
+        status = self._check_vm2_status(info1, info2)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [status]
+        return self._push_to_outputs(keys, values)
+
+    def _check_vm2_status(self, info1, info2):
+        if len(info1['pinning']) != 1 or len(info2['pinning']) != 1:
+            return False
+
+        for i in info1['vcpupin']:
+            for j in i['cpuset'].split(','):
+                if j not in self.cpu_set.split(','):
+                    return False
+
+        for i in info2['vcpupin']:
+            for j in i['cpuset'].split(','):
+                if j not in self.cpu_set.split(','):
+                    return False
+
+        return True
diff --git a/yardstick/benchmark/scenarios/lib/check_value.py b/yardstick/benchmark/scenarios/lib/check_value.py
new file mode 100644
index 000000000..759076068
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/check_value.py
@@ -0,0 +1,58 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class CheckValue(base.Scenario):
+    """Check values between value1 and value2
+
+    options:
+        operator: equal(eq) and not equal(ne)
+        value1:
+        value2:
+    output: check_result
+    """
+
+    __scenario_type__ = "CheckValue"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg['options']
+
+    def run(self, result):
+        """execute the test"""
+
+        op = self.options.get("operator")
+        LOG.debug("options=%s", self.options)
+        value1 = str(self.options.get("value1"))
+        value2 = str(self.options.get("value2"))
+        check_result = "PASS"
+        if op == "eq" and value1 != value2:
+            LOG.info("value1=%s, value2=%s, error: should equal!!!", value1,
+                     value2)
+            check_result = "FAIL"
+            assert value1 == value2, "Error %s!=%s" % (value1, value2)
+        elif op == "ne" and value1 == value2:
+            LOG.info("value1=%s, value2=%s, error: should not equal!!!",
+                     value1, value2)
+            check_result = "FAIL"
+            assert value1 != value2, "Error %s==%s" % (value1, value2)
+        LOG.info("Check result is %s", check_result)
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [check_result]
+        return self._push_to_outputs(keys, values)
diff --git a/yardstick/benchmark/scenarios/lib/get_migrate_target_host.py b/yardstick/benchmark/scenarios/lib/get_migrate_target_host.py
new file mode 100644
index 000000000..c19d96d68
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_migrate_target_host.py
@@ -0,0 +1,56 @@
+
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.common import openstack_utils
+from yardstick.common.utils import change_obj_to_dict
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class GetMigrateTargetHost(base.Scenario):
+    """Get a migrate target host according server
+    """
+
+    __scenario_type__ = "GetMigrateTargetHost"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+
+        self.options = self.scenario_cfg.get('options', {})
+        default_instance_id = self.options.get('server', {}).get('id', '')
+        self.instance_id = self.options.get('server_id', default_instance_id)
+
+        self.nova_client = openstack_utils.get_nova_client()
+
+    def run(self, result):
+        current_host = self._get_current_host_name(self.instance_id)
+        target_host = self._get_migrate_host(current_host)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [target_host]
+        return self._push_to_outputs(keys, values)
+
+    def _get_current_host_name(self, server_id):
+
+        return change_obj_to_dict(self.nova_client.servers.get(server_id))['OS-EXT-SRV-ATTR:host']
+
+    def _get_migrate_host(self, current_host):
+        hosts = self.nova_client.hosts.list_all()
+        compute_hosts = [a.host for a in hosts if a.service == 'compute']
+        for host in compute_hosts:
+            if host.strip() != current_host.strip():
+                return host
diff --git a/yardstick/benchmark/scenarios/lib/get_numa_info.py b/yardstick/benchmark/scenarios/lib/get_numa_info.py
new file mode 100644
index 000000000..4e4a44d95
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_numa_info.py
@@ -0,0 +1,79 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+import os
+
+import yaml
+from xml.etree import ElementTree as ET
+
+from yardstick import ssh
+from yardstick.benchmark.scenarios import base
+from yardstick.common import constants as consts
+from yardstick.common.utils import change_obj_to_dict
+from yardstick.common.openstack_utils import get_nova_client
+from yardstick.common.task_template import TaskTemplate
+
+LOG = logging.getLogger(__name__)
+
+
+class GetNumaInfo(base.Scenario):
+    """
+    Execute a live migration for two hosts
+
+    """
+
+    __scenario_type__ = "GetNumaInfo"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+
+        server = self.options['server']
+        self.server_id = server['id']
+        self.host = self._get_current_host_name(self.server_id)
+
+        node_file = os.path.join(consts.YARDSTICK_ROOT_PATH,
+                                 self.options.get('file'))
+
+        with open(node_file) as f:
+            nodes = yaml.safe_load(TaskTemplate.render(f.read()))
+        self.nodes = {a['host_name']: a for a in nodes['nodes']}
+
+    def run(self, result):
+        numa_info = self._check_numa_node(self.server_id, self.host)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [numa_info]
+        return self._push_to_outputs(keys, values)
+
+    def _get_current_host_name(self, server_id):
+
+        return change_obj_to_dict(get_nova_client().servers.get(server_id))['OS-EXT-SRV-ATTR:host']
+
+    def _get_host_client(self, node_name):
+        self.host_client = ssh.SSH.from_node(self.nodes.get(node_name))
+        self.host_client.wait(timeout=600)
+
+    def _check_numa_node(self, server_id, host):
+        self._get_host_client(host)
+
+        cmd = "sudo virsh dumpxml %s" % server_id
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.host_client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        root = ET.fromstring(stdout)
+        vcpupin = [a.attrib for a in root.iter('vcpupin')]
+        pinning = [a.attrib for a in root.iter('memnode')]
+        return {"pinning": pinning, 'vcpupin': vcpupin}
diff --git a/yardstick/benchmark/scenarios/lib/get_server.py b/yardstick/benchmark/scenarios/lib/get_server.py
new file mode 100644
index 000000000..fcf47c80d
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_server.py
@@ -0,0 +1,83 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+import yardstick.common.openstack_utils as op_utils
+
+LOG = logging.getLogger(__name__)
+
+
+class GetServer(base.Scenario):
+    """Get a server instance
+
+  Parameters
+    server_id - ID of the server
+        type:    string
+        unit:    N/A
+        default: null
+    server_name - name of the server
+        type:    string
+        unit:    N/A
+        default: null
+
+    Either server_id or server_name is required.
+
+  Outputs
+    rc - response code of getting server instance
+        0 for success
+        1 for failure
+        type:    int
+        unit:    N/A
+    server - instance of the server
+        type:    dict
+        unit:    N/A
+    """
+
+    __scenario_type__ = "GetServer"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+
+        self.server_id = self.options.get("server_id")
+        if self.server_id:
+            LOG.debug('Server id is %s', self.server_id)
+
+        default_name = self.scenario_cfg.get('host',
+                                             self.scenario_cfg.get('target'))
+        self.server_name = self.options.get('server_name', default_name)
+        if self.server_name:
+            LOG.debug('Server name is %s', self.server_name)
+
+        self.nova_client = op_utils.get_nova_client()
+
+    def run(self, result):
+        """execute the test"""
+
+        if self.server_id:
+            server = self.nova_client.servers.get(self.server_id)
+        else:
+            server = op_utils.get_server_by_name(self.server_name)
+
+        keys = self.scenario_cfg.get('output', '').split()
+
+        if server:
+            LOG.info("Get server successful!")
+            values = [0, self._change_obj_to_dict(server)]
+        else:
+            LOG.info("Get server failed!")
+            values = [1]
+
+        return self._push_to_outputs(keys, values)
diff --git a/yardstick/benchmark/scenarios/lib/get_server_ip.py b/yardstick/benchmark/scenarios/lib/get_server_ip.py
new file mode 100644
index 000000000..1eeeb7fca
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_server_ip.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class GetServerIp(base.Scenario):
+    """Get a server by name"""
+
+    __scenario_type__ = "GetServerIp"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+        self.ip_type = self.options.get('ip_type', "floating")
+
+    def run(self, result):
+        server = self.options.get('server', {})
+        ip = next(n['addr'] for k, v in server['addresses'].items()
+                  for n in v if n['OS-EXT-IPS:type'] == self.ip_type)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [ip]
+        return self._push_to_outputs(keys, values)
diff --git a/yardstick/benchmark/scenarios/lib/migrate.py b/yardstick/benchmark/scenarios/lib/migrate.py
new file mode 100644
index 000000000..116bae69e
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/migrate.py
@@ -0,0 +1,155 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+import subprocess
+import threading
+import time
+
+from datetime import datetime
+import ping
+
+from yardstick.common import openstack_utils
+from yardstick.common.utils import change_obj_to_dict
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+TIMEOUT = 0.05
+PACKAGE_SIZE = 64
+
+
+class Migrate(base.Scenario):       # pragma: no cover
+    """
+    Execute a live migration for two hosts
+
+    """
+
+    __scenario_type__ = "Migrate"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+
+        self.nova_client = openstack_utils.get_nova_client()
+
+    def run(self, result):
+        default_instance_id = self.options.get('server', {}).get('id', '')
+        instance_id = self.options.get('server_id', default_instance_id)
+        LOG.info('Instance id is %s', instance_id)
+
+        target_host = self.options.get('host')
+        LOG.info('Target host is %s', target_host)
+
+        instance_ip = self.options.get('server_ip')
+        if instance_ip:
+            LOG.info('Instance ip is %s', instance_ip)
+
+            self._ping_until_connected(instance_ip)
+            LOG.info('Instance is connected')
+
+            LOG.debug('Start to ping instance')
+            ping_thread = self._do_ping_task(instance_ip)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        try:
+            LOG.info('Start to migrate')
+            self._do_migrate(instance_id, target_host)
+        except Exception as e:
+            return self._push_to_outputs(keys, [1, str(e).split('.')[0]])
+        else:
+            migrate_time = self._get_migrate_time(instance_id)
+            LOG.info('Migration time is %s s', migrate_time)
+
+            current_host = self._get_current_host_name(instance_id)
+            LOG.info('Current host is %s', current_host)
+            if current_host.strip() != target_host.strip():
+                LOG.error('current_host not equal to target_host')
+                values = [1, 'current_host not equal to target_host']
+                return self._push_to_outputs(keys, values)
+
+        if instance_ip:
+            ping_thread.flag = False
+            ping_thread.join()
+
+            downtime = ping_thread.get_delay()
+            LOG.info('Downtime is %s s', downtime)
+
+            values = [0, migrate_time, downtime]
+            return self._push_to_outputs(keys, values)
+        else:
+            values = [0, migrate_time]
+            return self._push_to_outputs(keys, values)
+
+    def _do_migrate(self, server_id, target_host):
+
+        cmd = ['nova', 'live-migration', server_id, target_host]
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        p.communicate()
+
+    def _ping_until_connected(self, instance_ip):
+        for i in range(3000):
+            res = ping.do_one(instance_ip, TIMEOUT, PACKAGE_SIZE)
+            if res:
+                break
+
+    def _do_ping_task(self, instance_ip):
+        ping_thread = PingThread(instance_ip)
+        ping_thread.start()
+        return ping_thread
+
+    def _get_current_host_name(self, server_id):
+
+        return change_obj_to_dict(self.nova_client.servers.get(server_id))['OS-EXT-SRV-ATTR:host']
+
+    def _get_migrate_time(self, server_id):
+        while True:
+            status = self.nova_client.servers.get(server_id).status.lower()
+            if status == 'migrating':
+                start_time = datetime.now()
+                break
+        LOG.debug('Instance status change to MIGRATING')
+
+        while True:
+            status = self.nova_client.servers.get(server_id).status.lower()
+            if status == 'active':
+                end_time = datetime.now()
+                break
+            if status == 'error':
+                LOG.error('Instance status is ERROR')
+                raise RuntimeError('The instance status is error')
+        LOG.debug('Instance status change to ACTIVE')
+
+        duration = end_time - start_time
+        return duration.seconds + duration.microseconds * 1.0 / 1e6
+
+
+class PingThread(threading.Thread):     # pragma: no cover
+
+    def __init__(self, target):
+        super(PingThread, self).__init__()
+        self.target = target
+        self.flag = True
+        self.delay = 0.0
+
+    def run(self):
+        count = 0
+        while self.flag:
+            res = ping.do_one(self.target, TIMEOUT, PACKAGE_SIZE)
+            if not res:
+                count += 1
+            time.sleep(0.01)
+        self.delay = (TIMEOUT + 0.01) * count
+
+    def get_delay(self):
+        return self.delay