diff options
| -rw-r--r-- | ansible/build_yardstick_image.yml | 9 | ||||
| -rw-r--r-- | ansible/roles/install_dpdk/vars/main.yml | 5 | ||||
| -rw-r--r-- | ansible/roles/install_dpdk_shared/vars/main.yml | 5 | ||||
| -rwxr-xr-x | tests/ci/load_images.sh | 17 | ||||
| -rw-r--r-- | yardstick/benchmark/contexts/standalone/model.py | 42 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/lib/delete_network.py | 24 | ||||
| -rw-r--r-- | yardstick/common/exceptions.py | 15 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/contexts/standalone/test_model.py | 76 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py | 23 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_openstack_utils.py | 7 |
10 files changed, 172 insertions, 51 deletions
diff --git a/ansible/build_yardstick_image.yml b/ansible/build_yardstick_image.yml index 0fc29a9e3..072c12c66 100644 --- a/ansible/build_yardstick_image.yml +++ b/ansible/build_yardstick_image.yml @@ -76,13 +76,8 @@ - name: Debug dump loop devices command: losetup -a - register: losetup_output ignore_errors: true - - debug: - var: losetup_output - verbosity: 2 - - name: delete loop devices for image file # use this because kpartx -dv will fail if raw_imgfile was delete # but in theory we could have deleted file still attached to loopback device? @@ -90,6 +85,10 @@ shell: losetup -O NAME,BACK-FILE | grep "{{ raw_imgfile_basename }}" | awk '{ print $1 }' | xargs -l1 losetup -v -d ignore_errors: true + - name: Debug dump loop devices again + command: losetup -a + ignore_errors: true + - name: delete {{ raw_imgfile }} file: path: "{{ raw_imgfile }}" diff --git a/ansible/roles/install_dpdk/vars/main.yml b/ansible/roles/install_dpdk/vars/main.yml index 5dec63776..957f47e99 100644 --- a/ansible/roles/install_dpdk/vars/main.yml +++ b/ansible/roles/install_dpdk/vars/main.yml @@ -1,5 +1,8 @@ --- -dpdk_make_arch: x86_64-native-linuxapp-gcc +dpdk_make_archs: + "amd64": "x86_64-native-linuxapp-gcc" + "arm64": "arm64-native-linuxapp-gcc" +dpdk_make_arch: "{{ dpdk_make_archs[YARD_IMG_ARCH] }}" dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra" hugetable_mount: /mnt/huge dpdk_devbind_tools: "{{ dpdk_path }}/tools/dpdk-devbind.py" diff --git a/ansible/roles/install_dpdk_shared/vars/main.yml b/ansible/roles/install_dpdk_shared/vars/main.yml index eadf35a03..b663cedd2 100644 --- a/ansible/roles/install_dpdk_shared/vars/main.yml +++ b/ansible/roles/install_dpdk_shared/vars/main.yml @@ -1,5 +1,8 @@ --- -dpdk_make_arch: x86_64-native-linuxapp-gcc +dpdk_make_archs: + "amd64": "x86_64-native-linuxapp-gcc" + "arm64": "arm64-native-linuxapp-gcc" +dpdk_make_arch: "{{ dpdk_make_archs[YARD_IMG_ARCH] }}" dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra" hugetable_mount: /mnt/huge dpdk_pmd_path: /usr/lib/dpdk-pmd/ diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh index dee675981..1e1591ce3 100755 --- a/tests/ci/load_images.sh +++ b/tests/ci/load_images.sh @@ -43,6 +43,12 @@ if [ "${YARD_IMG_ARCH}" == "arm64" ]; then fi fi +cleanup_loopbacks() { + # try again to cleanup loopbacks in case of error + losetup -a + losetup -O NAME,BACK-FILE | awk '/yardstick/ { print $1 }' | xargs -l1 losetup -v -d || true +} + build_yardstick_image() { echo @@ -56,6 +62,7 @@ build_yardstick_image() # Build the image. Retry once if the build fails $cmd || $cmd + cleanup_loopbacks if [ ! -f "${RAW_IMAGE}" ]; then echo "Failed building RAW image" exit 1 @@ -70,16 +77,20 @@ build_yardstick_image() -e YARD_IMG_ARCH=${YARD_IMG_ARCH} \ -vvv -i inventory.ini build_yardstick_image.yml + cleanup_loopbacks if [ ! -f "${QCOW_IMAGE}" ]; then echo "Failed building QCOW image" exit 1 fi fi - if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then + # DPDK compile is not enabled for arm64 yet so disable for now + # JIRA: YARSTICK-1124 + if [[ ! -f "${QCOW_NSB_IMAGE}" && ${DEPLOY_SCENARIO} == *[_-]ovs_dpdk[_-]* && "${YARD_IMG_ARCH}" != "arm64" ]]; then ansible-playbook \ -e img_property="nsb" \ -e YARD_IMG_ARCH=${YARD_IMG_ARCH} \ -vvv -i inventory.ini build_yardstick_image.yml + cleanup_loopbacks if [ ! -f "${QCOW_NSB_IMAGE}" ]; then echo "Failed building QCOW NSB image" exit 1 @@ -122,7 +133,9 @@ load_yardstick_image() ${EXTRA_PARAMS} \ --file ${QCOW_IMAGE} \ yardstick-image) - if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then + # DPDK compile is not enabled for arm64 yet so disable NSB images for now + # JIRA: YARSTICK-1124 + if [[ $DEPLOY_SCENARIO == *[_-]ovs_dpdk[_-]* && "${YARD_IMG_ARCH}" != "arm64" ]]; then nsb_output=$(eval openstack ${SECURE} image create \ --public \ --disk-format qcow2 \ diff --git a/yardstick/benchmark/contexts/standalone/model.py b/yardstick/benchmark/contexts/standalone/model.py index f18d090d8..4d43f2611 100644 --- a/yardstick/benchmark/contexts/standalone/model.py +++ b/yardstick/benchmark/contexts/standalone/model.py @@ -232,14 +232,40 @@ class Libvirt(object): return ET.tostring(root) @staticmethod - def create_snapshot_qemu(connection, index, vm_image): - # build snapshot image - image = "/var/lib/libvirt/images/%s.qcow2" % index - connection.execute("rm %s" % image) - qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s" - connection.execute(qemu_template % (vm_image, image)) - - return image + def create_snapshot_qemu(connection, index, base_image): + """Create the snapshot image for a VM using a base image + + :param connection: SSH connection to the remote host + :param index: index of the VM to be spawn + :param base_image: path of the VM base image in the remote host + :return: snapshot image path + """ + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + connection.execute('rm -- "%s"' % vm_image) + status, _, _ = connection.execute('test -r %s' % base_image) + if status: + if not os.access(base_image, os.R_OK): + raise exceptions.LibvirtQemuImageBaseImageNotPresent( + vm_image=vm_image, base_image=base_image) + # NOTE(ralonsoh): done in two steps to avoid root permission + # issues. + LOG.info('Copy %s from execution host to remote host', base_image) + file_name = os.path.basename(os.path.normpath(base_image)) + connection.put_file(base_image, '/tmp/%s' % file_name) + status, _, error = connection.execute( + 'mv -- "/tmp/%s" "%s"' % (file_name, base_image)) + if status: + raise exceptions.LibvirtQemuImageCreateError( + vm_image=vm_image, base_image=base_image, error=error) + + LOG.info('Convert image %s to %s', base_image, vm_image) + qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + status, _, error = connection.execute(qemu_cmd) + if status: + raise exceptions.LibvirtQemuImageCreateError( + vm_image=vm_image, base_image=base_image, error=error) + return vm_image @classmethod def build_vm_xml(cls, connection, flavor, vm_name, index): diff --git a/yardstick/benchmark/scenarios/lib/delete_network.py b/yardstick/benchmark/scenarios/lib/delete_network.py index 2e8b595f9..8874e8b1e 100644 --- a/yardstick/benchmark/scenarios/lib/delete_network.py +++ b/yardstick/benchmark/scenarios/lib/delete_network.py @@ -10,7 +10,8 @@ import logging from yardstick.benchmark.scenarios import base -import yardstick.common.openstack_utils as op_utils +from yardstick.common import openstack_utils +from yardstick.common import exceptions LOG = logging.getLogger(__name__) @@ -24,11 +25,11 @@ class DeleteNetwork(base.Scenario): def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg - self.options = self.scenario_cfg['options'] + self.options = self.scenario_cfg["options"] - self.network_id = self.options.get("network_id", None) + self.network_name_or_id = self.options["network_name_or_id"] - self.shade_client = op_utils.get_shade_client() + self.shade_client = openstack_utils.get_shade_client() self.setup_done = False @@ -43,12 +44,13 @@ class DeleteNetwork(base.Scenario): if not self.setup_done: self.setup() - status = op_utils.delete_neutron_net(self.shade_client, - network_id=self.network_id) - if status: - result.update({"delete_network": 1}) - LOG.info("Delete network successful!") - else: + status = openstack_utils.delete_neutron_net(self.shade_client, + self.network_name_or_id) + + if not status: result.update({"delete_network": 0}) LOG.error("Delete network failed!") - return status + raise exceptions.ScenarioDeleteNetworkError + + result.update({"delete_network": 1}) + LOG.info("Delete network successful!") diff --git a/yardstick/common/exceptions.py b/yardstick/common/exceptions.py index 868308b55..ec21c335b 100644 --- a/yardstick/common/exceptions.py +++ b/yardstick/common/exceptions.py @@ -117,6 +117,17 @@ class LibvirtCreateError(YardstickException): message = 'Error creating the virtual machine. Error: %(error)s.' +class LibvirtQemuImageBaseImageNotPresent(YardstickException): + message = ('Error creating the qemu image for %(vm_image)s. Base image: ' + '%(base_image)s. Base image not present in execution host or ' + 'remote host.') + + +class LibvirtQemuImageCreateError(YardstickException): + message = ('Error creating the qemu image for %(vm_image)s. Base image: ' + '%(base_image)s. Error: %(error)s.') + + class ScenarioConfigContextNameNotFound(YardstickException): message = 'Context name "%(context_name)s" not found' @@ -175,3 +186,7 @@ class ScenarioDeleteFloatingIPError(YardstickException): class ScenarioCreateSecurityGroupError(YardstickException): message = 'Create Neutron Security Group Scenario failed' + + +class ScenarioDeleteNetworkError(YardstickException): + message = 'Delete Neutron Network Scenario failed' diff --git a/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py b/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py index b1dcee209..7078c89b2 100644 --- a/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py +++ b/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py @@ -13,11 +13,11 @@ # limitations under the License. import copy -import mock import os -import unittest import uuid +import mock +import unittest from xml.etree import ElementTree from yardstick import ssh @@ -172,14 +172,70 @@ class ModelLibvirtTestCase(unittest.TestCase): interface_address.get('function')) def test_create_snapshot_qemu(self): - result = "/var/lib/libvirt/images/0.qcow2" - with mock.patch("yardstick.ssh.SSH") as ssh: - ssh_mock = mock.Mock(autospec=ssh.SSH) - ssh_mock.execute = \ - mock.Mock(return_value=(0, "a", "")) - ssh.return_value = ssh_mock - image = model.Libvirt.create_snapshot_qemu(ssh_mock, "0", "ubuntu.img") - self.assertEqual(image, result) + self.mock_ssh.execute = mock.Mock(return_value=(0, 0, 0)) + index = 1 + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + base_image = '/tmp/base_image' + + model.Libvirt.create_snapshot_qemu(self.mock_ssh, index, base_image) + self.mock_ssh.execute.assert_has_calls([ + mock.call('rm -- "%s"' % vm_image), + mock.call('test -r %s' % base_image), + mock.call('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + ]) + + @mock.patch.object(os.path, 'basename', return_value='base_image') + @mock.patch.object(os.path, 'normpath') + @mock.patch.object(os, 'access', return_value=True) + def test_create_snapshot_qemu_no_image_remote(self, + mock_os_access, mock_normpath, mock_basename): + self.mock_ssh.execute = mock.Mock( + side_effect=[(0, 0, 0), (1, 0, 0), (0, 0, 0), (0, 0, 0)]) + index = 1 + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + base_image = '/tmp/base_image' + mock_normpath.return_value = base_image + + model.Libvirt.create_snapshot_qemu(self.mock_ssh, index, base_image) + self.mock_ssh.execute.assert_has_calls([ + mock.call('rm -- "%s"' % vm_image), + mock.call('test -r %s' % base_image), + mock.call('mv -- "/tmp/%s" "%s"' % ('base_image', base_image)), + mock.call('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + ]) + mock_os_access.assert_called_once_with(base_image, os.R_OK) + mock_normpath.assert_called_once_with(base_image) + mock_basename.assert_has_calls([mock.call(base_image)]) + self.mock_ssh.put_file.assert_called_once_with(base_image, + '/tmp/base_image') + + @mock.patch.object(os, 'access', return_value=False) + def test_create_snapshot_qemu_no_image_local(self, mock_os_access): + self.mock_ssh.execute = mock.Mock(side_effect=[(0, 0, 0), (1, 0, 0)]) + base_image = '/tmp/base_image' + + with self.assertRaises(exceptions.LibvirtQemuImageBaseImageNotPresent): + model.Libvirt.create_snapshot_qemu(self.mock_ssh, 3, base_image) + mock_os_access.assert_called_once_with(base_image, os.R_OK) + + def test_create_snapshot_qemu_error_qemuimg_command(self): + self.mock_ssh.execute = mock.Mock( + side_effect=[(0, 0, 0), (0, 0, 0), (1, 0, 0)]) + index = 1 + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + base_image = '/tmp/base_image' + + with self.assertRaises(exceptions.LibvirtQemuImageCreateError): + model.Libvirt.create_snapshot_qemu(self.mock_ssh, index, + base_image) + self.mock_ssh.execute.assert_has_calls([ + mock.call('rm -- "%s"' % vm_image), + mock.call('test -r %s' % base_image), + mock.call('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + ]) @mock.patch.object(model.Libvirt, 'pin_vcpu_for_perf', return_value='4,5') @mock.patch.object(model.Libvirt, 'create_snapshot_qemu', diff --git a/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py b/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py index aef99ee94..b6dbf4791 100644 --- a/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py +++ b/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py @@ -11,7 +11,8 @@ from oslo_utils import uuidutils import unittest import mock -import yardstick.common.openstack_utils as op_utils +from yardstick.common import openstack_utils +from yardstick.common import exceptions from yardstick.benchmark.scenarios.lib import delete_network @@ -19,16 +20,17 @@ class DeleteNetworkTestCase(unittest.TestCase): def setUp(self): self._mock_delete_neutron_net = mock.patch.object( - op_utils, 'delete_neutron_net') + openstack_utils, "delete_neutron_net") self.mock_delete_neutron_net = self._mock_delete_neutron_net.start() self._mock_get_shade_client = mock.patch.object( - op_utils, 'get_shade_client') + openstack_utils, "get_shade_client") self.mock_get_shade_client = self._mock_get_shade_client.start() - self._mock_log = mock.patch.object(delete_network, 'LOG') + self._mock_log = mock.patch.object(delete_network, "LOG") self.mock_log = self._mock_log.start() - _uuid = uuidutils.generate_uuid() - self.args = {'options': {'network_id': _uuid}} - self._del_obj = delete_network.DeleteNetwork(self.args, mock.ANY) + self.args = {"options": {"network_name_or_id": ( + uuidutils.generate_uuid())}} + self.result = {} + self.del_obj = delete_network.DeleteNetwork(self.args, mock.ANY) self.addCleanup(self._stop_mock) @@ -39,11 +41,14 @@ class DeleteNetworkTestCase(unittest.TestCase): def test_run(self): self.mock_delete_neutron_net.return_value = True - self.assertTrue(self._del_obj.run({})) + self.assertIsNone(self.del_obj.run(self.result)) + self.assertEqual({"delete_network": 1}, self.result) self.mock_log.info.assert_called_once_with( "Delete network successful!") def test_run_fail(self): self.mock_delete_neutron_net.return_value = False - self.assertFalse(self._del_obj.run({})) + with self.assertRaises(exceptions.ScenarioDeleteNetworkError): + self.del_obj.run(self.result) + self.assertEqual({"delete_network": 0}, self.result) self.mock_log.error.assert_called_once_with("Delete network failed!") diff --git a/yardstick/tests/unit/common/test_openstack_utils.py b/yardstick/tests/unit/common/test_openstack_utils.py index 3abd39668..f03f2516c 100644 --- a/yardstick/tests/unit/common/test_openstack_utils.py +++ b/yardstick/tests/unit/common/test_openstack_utils.py @@ -39,18 +39,17 @@ class DeleteNeutronNetTestCase(unittest.TestCase): def setUp(self): self.mock_shade_client = mock.Mock() - self.mock_shade_client.delete_network = mock.Mock() def test_delete_neutron_net(self): self.mock_shade_client.delete_network.return_value = True output = openstack_utils.delete_neutron_net(self.mock_shade_client, - 'network_id') + 'network_name_or_id') self.assertTrue(output) def test_delete_neutron_net_fail(self): self.mock_shade_client.delete_network.return_value = False output = openstack_utils.delete_neutron_net(self.mock_shade_client, - 'network_id') + 'network_name_or_id') self.assertFalse(output) @mock.patch.object(openstack_utils, 'log') @@ -58,7 +57,7 @@ class DeleteNeutronNetTestCase(unittest.TestCase): self.mock_shade_client.delete_network.side_effect = ( exc.OpenStackCloudException('error message')) output = openstack_utils.delete_neutron_net(self.mock_shade_client, - 'network_id') + 'network_name_or_id') self.assertFalse(output) mock_logger.error.assert_called_once() |