Merge "Test Case: OPNFV_YARDSTICK_TC023: VM availability during live migration"

10 files changed, 599 insertions, 0 deletions

diff --git a/yardstick/benchmark/scenarios/base.py b/yardstick/benchmark/scenarios/base.py
index 5d3c36c38..3cb138dd8 100644
--- a/yardstick/benchmark/scenarios/base.py
+++ b/yardstick/benchmark/scenarios/base.py
@@ -63,3 +63,15 @@ class Scenario(object):
                 return scenario.__module__ + "." + scenario.__name__
 
         raise RuntimeError("No such scenario type %s" % scenario_type)
+
+    def _push_to_outputs(self, keys, values):
+        return dict(zip(keys, values))
+
+    def _change_obj_to_dict(self, obj):
+        dic = {}
+        for k, v in vars(obj).items():
+            try:
+                vars(v)
+            except TypeError:
+                dic[k] = v
+        return dic
diff --git a/yardstick/benchmark/scenarios/lib/__init__.py b/yardstick/benchmark/scenarios/lib/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/__init__.py
diff --git a/yardstick/benchmark/scenarios/lib/add_memory_load.py b/yardstick/benchmark/scenarios/lib/add_memory_load.py
new file mode 100644
index 000000000..26cf140d1
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/add_memory_load.py
@@ -0,0 +1,57 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class AddMemoryLoad(base.Scenario):
+    """Add memory load in server
+    """
+
+    __scenario_type__ = "AddMemoryLoad"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+
+        self.options = scenario_cfg.get('options', {})
+
+        self.client = ssh.SSH.from_node(self.context_cfg['host'])
+        self.client.wait(timeout=600)
+
+    def run(self, result):
+        self._add_load()
+
+    def _add_load(self):
+        try:
+            memory_load = self.options['memory_load']
+        except KeyError:
+            LOG.error('memory_load parameter must be provided')
+        else:
+            if float(memory_load) == 0:
+                return
+            cmd = 'free | awk "/Mem/ {print $2}"'
+            code, stdout, stderr = self.client.execute(cmd)
+            total = int(stdout.split()[1])
+            used = int(stdout.split()[2])
+            remain_memory = total * float(memory_load) - used
+            if remain_memory > 0:
+                count = remain_memory / 1024 / 128
+                LOG.info('Add %s vm load', count)
+                if count != 0:
+                    cmd = 'stress -t 10 -m {} --vm-keep'.format(count)
+                    self.client.execute(cmd)
diff --git a/yardstick/benchmark/scenarios/lib/check_numa_info.py b/yardstick/benchmark/scenarios/lib/check_numa_info.py
new file mode 100644
index 000000000..59a47547e
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/check_numa_info.py
@@ -0,0 +1,61 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class CheckNumaInfo(base.Scenario):
+    """
+    Execute a live migration for two hosts
+
+    """
+
+    __scenario_type__ = "CheckNumaInfo"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+
+        self.options = self.scenario_cfg.get('options', {})
+
+        self.cpu_set = self.options.get('cpu_set', '1,2,3,4,5,6')
+
+    def run(self, result):
+        info1 = self.options.get('info1')
+        info2 = self.options.get('info2')
+        LOG.debug('Origin numa info: %s', info1)
+        LOG.debug('Current numa info: %s', info2)
+        status = self._check_vm2_status(info1, info2)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [status]
+        return self._push_to_outputs(keys, values)
+
+    def _check_vm2_status(self, info1, info2):
+        if len(info1['pinning']) != 1 or len(info2['pinning']) != 1:
+            return False
+
+        for i in info1['vcpupin']:
+            for j in i['cpuset'].split(','):
+                if j not in self.cpu_set.split(','):
+                    return False
+
+        for i in info2['vcpupin']:
+            for j in i['cpuset'].split(','):
+                if j not in self.cpu_set.split(','):
+                    return False
+
+        return True
diff --git a/yardstick/benchmark/scenarios/lib/check_value.py b/yardstick/benchmark/scenarios/lib/check_value.py
new file mode 100644
index 000000000..759076068
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/check_value.py
@@ -0,0 +1,58 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class CheckValue(base.Scenario):
+    """Check values between value1 and value2
+
+    options:
+        operator: equal(eq) and not equal(ne)
+        value1:
+        value2:
+    output: check_result
+    """
+
+    __scenario_type__ = "CheckValue"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg['options']
+
+    def run(self, result):
+        """execute the test"""
+
+        op = self.options.get("operator")
+        LOG.debug("options=%s", self.options)
+        value1 = str(self.options.get("value1"))
+        value2 = str(self.options.get("value2"))
+        check_result = "PASS"
+        if op == "eq" and value1 != value2:
+            LOG.info("value1=%s, value2=%s, error: should equal!!!", value1,
+                     value2)
+            check_result = "FAIL"
+            assert value1 == value2, "Error %s!=%s" % (value1, value2)
+        elif op == "ne" and value1 == value2:
+            LOG.info("value1=%s, value2=%s, error: should not equal!!!",
+                     value1, value2)
+            check_result = "FAIL"
+            assert value1 != value2, "Error %s==%s" % (value1, value2)
+        LOG.info("Check result is %s", check_result)
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [check_result]
+        return self._push_to_outputs(keys, values)
diff --git a/yardstick/benchmark/scenarios/lib/get_migrate_target_host.py b/yardstick/benchmark/scenarios/lib/get_migrate_target_host.py
new file mode 100644
index 000000000..c19d96d68
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_migrate_target_host.py
@@ -0,0 +1,56 @@
+
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.common import openstack_utils
+from yardstick.common.utils import change_obj_to_dict
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class GetMigrateTargetHost(base.Scenario):
+    """Get a migrate target host according server
+    """
+
+    __scenario_type__ = "GetMigrateTargetHost"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+
+        self.options = self.scenario_cfg.get('options', {})
+        default_instance_id = self.options.get('server', {}).get('id', '')
+        self.instance_id = self.options.get('server_id', default_instance_id)
+
+        self.nova_client = openstack_utils.get_nova_client()
+
+    def run(self, result):
+        current_host = self._get_current_host_name(self.instance_id)
+        target_host = self._get_migrate_host(current_host)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [target_host]
+        return self._push_to_outputs(keys, values)
+
+    def _get_current_host_name(self, server_id):
+
+        return change_obj_to_dict(self.nova_client.servers.get(server_id))['OS-EXT-SRV-ATTR:host']
+
+    def _get_migrate_host(self, current_host):
+        hosts = self.nova_client.hosts.list_all()
+        compute_hosts = [a.host for a in hosts if a.service == 'compute']
+        for host in compute_hosts:
+            if host.strip() != current_host.strip():
+                return host
diff --git a/yardstick/benchmark/scenarios/lib/get_numa_info.py b/yardstick/benchmark/scenarios/lib/get_numa_info.py
new file mode 100644
index 000000000..4e4a44d95
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_numa_info.py
@@ -0,0 +1,79 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+import os
+
+import yaml
+from xml.etree import ElementTree as ET
+
+from yardstick import ssh
+from yardstick.benchmark.scenarios import base
+from yardstick.common import constants as consts
+from yardstick.common.utils import change_obj_to_dict
+from yardstick.common.openstack_utils import get_nova_client
+from yardstick.common.task_template import TaskTemplate
+
+LOG = logging.getLogger(__name__)
+
+
+class GetNumaInfo(base.Scenario):
+    """
+    Execute a live migration for two hosts
+
+    """
+
+    __scenario_type__ = "GetNumaInfo"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+
+        server = self.options['server']
+        self.server_id = server['id']
+        self.host = self._get_current_host_name(self.server_id)
+
+        node_file = os.path.join(consts.YARDSTICK_ROOT_PATH,
+                                 self.options.get('file'))
+
+        with open(node_file) as f:
+            nodes = yaml.safe_load(TaskTemplate.render(f.read()))
+        self.nodes = {a['host_name']: a for a in nodes['nodes']}
+
+    def run(self, result):
+        numa_info = self._check_numa_node(self.server_id, self.host)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [numa_info]
+        return self._push_to_outputs(keys, values)
+
+    def _get_current_host_name(self, server_id):
+
+        return change_obj_to_dict(get_nova_client().servers.get(server_id))['OS-EXT-SRV-ATTR:host']
+
+    def _get_host_client(self, node_name):
+        self.host_client = ssh.SSH.from_node(self.nodes.get(node_name))
+        self.host_client.wait(timeout=600)
+
+    def _check_numa_node(self, server_id, host):
+        self._get_host_client(host)
+
+        cmd = "sudo virsh dumpxml %s" % server_id
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.host_client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        root = ET.fromstring(stdout)
+        vcpupin = [a.attrib for a in root.iter('vcpupin')]
+        pinning = [a.attrib for a in root.iter('memnode')]
+        return {"pinning": pinning, 'vcpupin': vcpupin}
diff --git a/yardstick/benchmark/scenarios/lib/get_server.py b/yardstick/benchmark/scenarios/lib/get_server.py
new file mode 100644
index 000000000..fcf47c80d
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_server.py
@@ -0,0 +1,83 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+import yardstick.common.openstack_utils as op_utils
+
+LOG = logging.getLogger(__name__)
+
+
+class GetServer(base.Scenario):
+    """Get a server instance
+
+  Parameters
+    server_id - ID of the server
+        type:    string
+        unit:    N/A
+        default: null
+    server_name - name of the server
+        type:    string
+        unit:    N/A
+        default: null
+
+    Either server_id or server_name is required.
+
+  Outputs
+    rc - response code of getting server instance
+        0 for success
+        1 for failure
+        type:    int
+        unit:    N/A
+    server - instance of the server
+        type:    dict
+        unit:    N/A
+    """
+
+    __scenario_type__ = "GetServer"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+
+        self.server_id = self.options.get("server_id")
+        if self.server_id:
+            LOG.debug('Server id is %s', self.server_id)
+
+        default_name = self.scenario_cfg.get('host',
+                                             self.scenario_cfg.get('target'))
+        self.server_name = self.options.get('server_name', default_name)
+        if self.server_name:
+            LOG.debug('Server name is %s', self.server_name)
+
+        self.nova_client = op_utils.get_nova_client()
+
+    def run(self, result):
+        """execute the test"""
+
+        if self.server_id:
+            server = self.nova_client.servers.get(self.server_id)
+        else:
+            server = op_utils.get_server_by_name(self.server_name)
+
+        keys = self.scenario_cfg.get('output', '').split()
+
+        if server:
+            LOG.info("Get server successful!")
+            values = [0, self._change_obj_to_dict(server)]
+        else:
+            LOG.info("Get server failed!")
+            values = [1]
+
+        return self._push_to_outputs(keys, values)
diff --git a/yardstick/benchmark/scenarios/lib/get_server_ip.py b/yardstick/benchmark/scenarios/lib/get_server_ip.py
new file mode 100644
index 000000000..1eeeb7fca
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/get_server_ip.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class GetServerIp(base.Scenario):
+    """Get a server by name"""
+
+    __scenario_type__ = "GetServerIp"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+        self.ip_type = self.options.get('ip_type', "floating")
+
+    def run(self, result):
+        server = self.options.get('server', {})
+        ip = next(n['addr'] for k, v in server['addresses'].items()
+                  for n in v if n['OS-EXT-IPS:type'] == self.ip_type)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        values = [ip]
+        return self._push_to_outputs(keys, values)
diff --git a/yardstick/benchmark/scenarios/lib/migrate.py b/yardstick/benchmark/scenarios/lib/migrate.py
new file mode 100644
index 000000000..116bae69e
--- /dev/null
+++ b/yardstick/benchmark/scenarios/lib/migrate.py
@@ -0,0 +1,155 @@
+# ############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# ############################################################################
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import logging
+import subprocess
+import threading
+import time
+
+from datetime import datetime
+import ping
+
+from yardstick.common import openstack_utils
+from yardstick.common.utils import change_obj_to_dict
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+TIMEOUT = 0.05
+PACKAGE_SIZE = 64
+
+
+class Migrate(base.Scenario):       # pragma: no cover
+    """
+    Execute a live migration for two hosts
+
+    """
+
+    __scenario_type__ = "Migrate"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.options = self.scenario_cfg.get('options', {})
+
+        self.nova_client = openstack_utils.get_nova_client()
+
+    def run(self, result):
+        default_instance_id = self.options.get('server', {}).get('id', '')
+        instance_id = self.options.get('server_id', default_instance_id)
+        LOG.info('Instance id is %s', instance_id)
+
+        target_host = self.options.get('host')
+        LOG.info('Target host is %s', target_host)
+
+        instance_ip = self.options.get('server_ip')
+        if instance_ip:
+            LOG.info('Instance ip is %s', instance_ip)
+
+            self._ping_until_connected(instance_ip)
+            LOG.info('Instance is connected')
+
+            LOG.debug('Start to ping instance')
+            ping_thread = self._do_ping_task(instance_ip)
+
+        keys = self.scenario_cfg.get('output', '').split()
+        try:
+            LOG.info('Start to migrate')
+            self._do_migrate(instance_id, target_host)
+        except Exception as e:
+            return self._push_to_outputs(keys, [1, str(e).split('.')[0]])
+        else:
+            migrate_time = self._get_migrate_time(instance_id)
+            LOG.info('Migration time is %s s', migrate_time)
+
+            current_host = self._get_current_host_name(instance_id)
+            LOG.info('Current host is %s', current_host)
+            if current_host.strip() != target_host.strip():
+                LOG.error('current_host not equal to target_host')
+                values = [1, 'current_host not equal to target_host']
+                return self._push_to_outputs(keys, values)
+
+        if instance_ip:
+            ping_thread.flag = False
+            ping_thread.join()
+
+            downtime = ping_thread.get_delay()
+            LOG.info('Downtime is %s s', downtime)
+
+            values = [0, migrate_time, downtime]
+            return self._push_to_outputs(keys, values)
+        else:
+            values = [0, migrate_time]
+            return self._push_to_outputs(keys, values)
+
+    def _do_migrate(self, server_id, target_host):
+
+        cmd = ['nova', 'live-migration', server_id, target_host]
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        p.communicate()
+
+    def _ping_until_connected(self, instance_ip):
+        for i in range(3000):
+            res = ping.do_one(instance_ip, TIMEOUT, PACKAGE_SIZE)
+            if res:
+                break
+
+    def _do_ping_task(self, instance_ip):
+        ping_thread = PingThread(instance_ip)
+        ping_thread.start()
+        return ping_thread
+
+    def _get_current_host_name(self, server_id):
+
+        return change_obj_to_dict(self.nova_client.servers.get(server_id))['OS-EXT-SRV-ATTR:host']
+
+    def _get_migrate_time(self, server_id):
+        while True:
+            status = self.nova_client.servers.get(server_id).status.lower()
+            if status == 'migrating':
+                start_time = datetime.now()
+                break
+        LOG.debug('Instance status change to MIGRATING')
+
+        while True:
+            status = self.nova_client.servers.get(server_id).status.lower()
+            if status == 'active':
+                end_time = datetime.now()
+                break
+            if status == 'error':
+                LOG.error('Instance status is ERROR')
+                raise RuntimeError('The instance status is error')
+        LOG.debug('Instance status change to ACTIVE')
+
+        duration = end_time - start_time
+        return duration.seconds + duration.microseconds * 1.0 / 1e6
+
+
+class PingThread(threading.Thread):     # pragma: no cover
+
+    def __init__(self, target):
+        super(PingThread, self).__init__()
+        self.target = target
+        self.flag = True
+        self.delay = 0.0
+
+    def run(self):
+        count = 0
+        while self.flag:
+            res = ping.do_one(self.target, TIMEOUT, PACKAGE_SIZE)
+            if not res:
+                count += 1
+            time.sleep(0.01)
+        self.delay = (TIMEOUT + 0.01) * count
+
+    def get_delay(self):
+        return self.delay