diff options
28 files changed, 622 insertions, 71 deletions
diff --git a/ansible/build_yardstick_image.yml b/ansible/build_yardstick_image.yml index 429ab88aa..c926af208 100644 --- a/ansible/build_yardstick_image.yml +++ b/ansible/build_yardstick_image.yml @@ -75,8 +75,9 @@ ignore_errors: true - name: Debug dump loop devices - command: losetup + command: losetup -a register: losetup_output + ignore_errors: true - debug: var: losetup_output diff --git a/ansible/roles/download_collectd/defaults/main.yml b/ansible/roles/download_collectd/defaults/main.yml index 9beaeb897..3f5632394 100644 --- a/ansible/roles/download_collectd/defaults/main.yml +++ b/ansible/roles/download_collectd/defaults/main.yml @@ -1,4 +1,4 @@ --- collectd_url: "https://github.com/collectd/collectd.git" collectd_dest: "{{ clone_dest }}/collectd" -collectd_version: "c870991a2d614e51c03c0da76e9aef997343551b" +collectd_version: "collectd-5.8" diff --git a/ansible/roles/download_dpdk/defaults/main.yml b/ansible/roles/download_dpdk/defaults/main.yml index fb42ed48a..d548280f5 100644 --- a/ansible/roles/download_dpdk/defaults/main.yml +++ b/ansible/roles/download_dpdk/defaults/main.yml @@ -1,10 +1,14 @@ --- dpdk_version: "17.02" -dpdk_url: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.zip" +dpdk_url: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz" dpdk_file: "{{ dpdk_url|basename }}" -dpdk_unarchive: "{{ dpdk_file|regex_replace('[.]zip$', '') }}" +dpdk_unarchive: "{{ dpdk_file|regex_replace('[.]tar[.]gz$', '') }}" dpdk_dest: "{{ clone_dest }}/" +#Note DPDK 17.08 17.11 and 18.02 are currently unsupported due to prox build issues dpdk_sha256s: - "17.02": "sha256:c675285d5eab8e7e8537a10bdae6f03c603caa80cb4bf5b055ddd482e3b7d67f" - "16.04": "sha256:65e587af439b21abf54c644f86ba5ce7ea65657b3272480dcd78b769d2aecac3" - "16.07": "sha256:46fee52f9b3ff54df869414c69f122aea8f744de5ed5f8fb275a22cc9cefe094" + "16.07": "sha256:d876e4b2a7101f28e7e345d3c88e66afe877d15f0159c19c5bc5bc26b7b7d788" + "17.02": "sha256:b07b546e910095174bdb6152bb0d7ce057cc4b79aaa74771aeee4e8a7219fb38" + "17.05": "sha256:763bfb7e1765efcc949e79d645dc9f1ebd16591431ba0db5ce22becd928dcd0a" + "17.08": "sha256:3a08addbff45c636538514e9a5838fb91ea557661a4c071e03a9a6987d46e5b6" #unsupported + "17.11": "sha256:77a727bb3834549985f291409c9a77a1e8be1c9329ce4c3eb19a22d1461022e4" #unsupported + "18.02": "sha256:f1210310fd5f01a3babe3a09d9b3e5a9db791c2ec6ecfbf94ade9f893a0632b8" #unsupported diff --git a/ansible/roles/install_civetweb/defaults/main.yml b/ansible/roles/install_civetweb/defaults/main.yml index 2235fe172..ed5ab27f2 100644 --- a/ansible/roles/install_civetweb/defaults/main.yml +++ b/ansible/roles/install_civetweb/defaults/main.yml @@ -15,6 +15,7 @@ civetweb_dest: "{{ clone_dest }}/civetweb" civetweb_build_dependencies: Debian: + - libjson-c-dev=0.11-4ubuntu2 - libjson0 - libjson0-dev - libssl-dev diff --git a/ansible/roles/install_dpdk/tasks/main.yml b/ansible/roles/install_dpdk/tasks/main.yml index e82ad8363..5bcfb50b1 100644 --- a/ansible/roles/install_dpdk/tasks/main.yml +++ b/ansible/roles/install_dpdk/tasks/main.yml @@ -116,7 +116,7 @@ - name: copy dpdk-devbind.py to correct location copy: - src: "{{ dpdk_devbind[dpdk_version] }}" + src: "{{ dpdk_devbind_usertools if dpdk_version|float >= 17.02 else dpdk_devbind_tools }}" dest: "{{ INSTALL_BIN_PATH }}/dpdk-devbind.py" remote_src: yes force: yes diff --git a/ansible/roles/install_dpdk/vars/main.yml b/ansible/roles/install_dpdk/vars/main.yml index 45bcc339b..5dec63776 100644 --- a/ansible/roles/install_dpdk/vars/main.yml +++ b/ansible/roles/install_dpdk/vars/main.yml @@ -2,8 +2,6 @@ dpdk_make_arch: x86_64-native-linuxapp-gcc dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra" hugetable_mount: /mnt/huge -dpdk_devbind: - "16.07": "{{ dpdk_path }}/tools/dpdk-devbind.py" - "17.02": "{{ dpdk_path }}/usertools/dpdk-devbind.py" - "17.04": "{{ dpdk_path }}/usertools/dpdk-devbind.py" +dpdk_devbind_tools: "{{ dpdk_path }}/tools/dpdk-devbind.py" +dpdk_devbind_usertools: "{{ dpdk_path }}/usertools/dpdk-devbind.py" dpdk_pmd_path: /usr/lib/dpdk-pmd/ diff --git a/ansible/roles/install_dpdk_shared/tasks/Debian.yml b/ansible/roles/install_dpdk_shared/tasks/Debian.yml index c77e4f96a..641d8f9cd 100755 --- a/ansible/roles/install_dpdk_shared/tasks/Debian.yml +++ b/ansible/roles/install_dpdk_shared/tasks/Debian.yml @@ -16,6 +16,7 @@ action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" with_items: - libpcap-dev + - libnuma-dev - name: Install kernel headers action: "{{ ansible_pkg_mgr }} name=linux-headers-{{ dpdk_kernel }} state=present" diff --git a/ansible/roles/install_dpdk_shared/tasks/main.yml b/ansible/roles/install_dpdk_shared/tasks/main.yml index 65954be04..94b9215bd 100644 --- a/ansible/roles/install_dpdk_shared/tasks/main.yml +++ b/ansible/roles/install_dpdk_shared/tasks/main.yml @@ -42,6 +42,15 @@ set_fact: RTE_KERNELDIR: "/lib/modules/{{ dpdk_kernel }}/build" +# make clean must be run here as DPDK-shared is a copy of the DPDK directory +# which has already run the make command +# no T= target for clean +- command: make -j {{ ansible_processor_vcpus }} clean O={{ dpdk_make_arch }} + args: + chdir: "{{ dpdk_shared_path }}" + environment: + RTE_KERNELDIR: "{{ RTE_KERNELDIR }}" + - command: make -j {{ ansible_processor_vcpus }} config T={{ dpdk_make_arch }} O={{ dpdk_make_arch }} args: chdir: "{{ dpdk_shared_path }}" @@ -85,13 +94,6 @@ regexp: '^CONFIG_RTE_EAL_PMD_PATH=""' line: 'CONFIG_RTE_EAL_PMD_PATH="{{ dpdk_pmd_path }}"' - # no T= target for clean -- command: make -j {{ ansible_processor_vcpus }} clean O={{ dpdk_make_arch }} - args: - chdir: "{{ dpdk_shared_path }}" - environment: - RTE_KERNELDIR: "{{ RTE_KERNELDIR }}" - # TODO: disable ASLR - command: make -j {{ ansible_processor_vcpus }} diff --git a/ansible/roles/install_dpdk_shared/vars/main.yml b/ansible/roles/install_dpdk_shared/vars/main.yml index 45bcc339b..eadf35a03 100644 --- a/ansible/roles/install_dpdk_shared/vars/main.yml +++ b/ansible/roles/install_dpdk_shared/vars/main.yml @@ -2,8 +2,4 @@ dpdk_make_arch: x86_64-native-linuxapp-gcc dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra" hugetable_mount: /mnt/huge -dpdk_devbind: - "16.07": "{{ dpdk_path }}/tools/dpdk-devbind.py" - "17.02": "{{ dpdk_path }}/usertools/dpdk-devbind.py" - "17.04": "{{ dpdk_path }}/usertools/dpdk-devbind.py" dpdk_pmd_path: /usr/lib/dpdk-pmd/ diff --git a/api/resources/v2/images.py b/api/resources/v2/images.py index 0c36a0a26..c3e5ee73e 100644 --- a/api/resources/v2/images.py +++ b/api/resources/v2/images.py @@ -18,8 +18,7 @@ from api.database.v2.handlers import V2ImageHandler from api.database.v2.handlers import V2EnvironmentHandler from yardstick.common.utils import result_handler from yardstick.common.utils import source_env -from yardstick.common.utils import change_obj_to_dict -from yardstick.common.openstack_utils import get_nova_client +from yardstick.common import openstack_utils from yardstick.common.openstack_utils import get_glance_client from yardstick.common import constants as consts @@ -47,39 +46,21 @@ class V2Images(ApiResource): def get(self): try: source_env(consts.OPENRC) - except Exception: + except OSError: return result_handler(consts.API_ERROR, 'source openrc error') - nova_client = get_nova_client() - try: - images_list = nova_client.images.list() - except Exception: + image_list = openstack_utils.list_images() + + if image_list is False: return result_handler(consts.API_ERROR, 'get images error') - else: - images = {i.name: self.get_info(change_obj_to_dict(i)) for i in images_list} + + images = {i.name: format_image_info(i) for i in image_list} return result_handler(consts.API_SUCCESS, {'status': 1, 'images': images}) def post(self): return self._dispatch_post() - def get_info(self, data): - try: - size = data['OS-EXT-IMG-SIZE:size'] - except KeyError: - size = None - else: - size = float(size) / 1024 / 1024 - - result = { - 'name': data.get('name', ''), - 'discription': data.get('description', ''), - 'size': size, - 'status': data.get('status'), - 'time': data.get('updated') - } - return result - def load_image(self, args): try: image_name = args['name'] @@ -268,7 +249,7 @@ class V2Images(ApiResource): r = requests.head(url) try: file_size = int(r.headers['content-length']) - except Exception: + except (TypeError, ValueError): return with open(path, 'wb') as f: @@ -303,14 +284,13 @@ class V2Image(ApiResource): except ValueError: return result_handler(consts.API_ERROR, 'no such image id') - nova_client = get_nova_client() - images = nova_client.images.list() + images = openstack_utils.list_images() try: image = next((i for i in images if i.name == image.name)) except StopIteration: pass - return_image = self.get_info(change_obj_to_dict(image)) + return_image = format_image_info(image) return_image['id'] = image_id return result_handler(consts.API_SUCCESS, {'image': return_image}) @@ -349,19 +329,16 @@ class V2Image(ApiResource): return result_handler(consts.API_SUCCESS, {'image': image_id}) - def get_info(self, data): - try: - size = data['OS-EXT-IMG-SIZE:size'] - except KeyError: - size = None - else: - size = float(size) / 1024 / 1024 - - result = { - 'name': data.get('name', ''), - 'description': data.get('description', ''), - 'size': size, - 'status': data.get('status'), - 'time': data.get('updated') - } - return result + +def format_image_info(image): + image_dict = {} + + if image is None: + return image_dict + + image_dict['name'] = image.name + image_dict['size'] = float(image.size) / 1024 / 1024 + image_dict['status'] = image.status.upper() + image_dict['time'] = image.updated_at + + return image_dict diff --git a/requirements.txt b/requirements.txt index 02545de1d..43f0e796c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,6 +37,7 @@ os-client-config==1.28.0 # OSI Approved Apache Software License osc-lib==1.7.0 # OSI Approved Apache Software License oslo.config==4.11.1 # OSI Approved Apache Software License oslo.i18n==3.17.0 # OSI Approved Apache Software License +oslo.messaging===5.30.2 # OSI Approved Apache Software License oslo.privsep===1.22.1 # OSI Approved Apache Software License oslo.serialization==2.20.1 # OSI Approved Apache Software License oslo.utils==3.28.0 # OSI Approved Apache Software License diff --git a/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash b/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash index 5a5dbc394..9f1804819 100644 --- a/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash @@ -18,7 +18,7 @@ OUTPUT_FILE=/tmp/unixbench-out.log # run unixbench test run_unixbench() { - cd /opt/tempT/UnixBench/ + cd /opt/tempT/UnixBench/UnixBench/ ./Run $OPTIONS > $OUTPUT_FILE } diff --git a/yardstick/common/exceptions.py b/yardstick/common/exceptions.py index 439b9cb1b..65e444071 100644 --- a/yardstick/common/exceptions.py +++ b/yardstick/common/exceptions.py @@ -64,6 +64,11 @@ class YardstickBannedModuleImported(YardstickException): message = 'Module "%(module)s" cannnot be imported. Reason: "%(reason)s"' +class PayloadMissingAttributes(YardstickException): + message = ('Error instantiating a Payload class, missing attributes: ' + '%(missing_attributes)s') + + class HeatTemplateError(YardstickException): """Error in Heat during the stack deployment""" message = ('Error in Heat during the creation of the OpenStack stack ' diff --git a/yardstick/common/messaging/__init__.py b/yardstick/common/messaging/__init__.py new file mode 100644 index 000000000..f0f012ec3 --- /dev/null +++ b/yardstick/common/messaging/__init__.py @@ -0,0 +1,36 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MQ is statically configured: +# - MQ service: RabbitMQ +# - user/password: yardstick/yardstick +# - host:port: localhost:5672 +MQ_USER = 'yardstick' +MQ_PASS = 'yardstick' +MQ_SERVICE = 'rabbit' +SERVER = 'localhost' +PORT = 5672 +TRANSPORT_URL = (MQ_SERVICE + '://' + MQ_USER + ':' + MQ_PASS + '@' + SERVER + + ':' + str(PORT) + '/') + +# RPC server. +RPC_SERVER_EXECUTOR = 'threading' + +# Topics. +RUNNER = 'runner' + +# Methods. +# RUNNER methods: +RUNNER_INFO = 'runner_info' +RUNNER_LOOP = 'runner_loop' diff --git a/yardstick/common/messaging/consumer.py b/yardstick/common/messaging/consumer.py new file mode 100644 index 000000000..a0feeb300 --- /dev/null +++ b/yardstick/common/messaging/consumer.py @@ -0,0 +1,85 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import logging + +from oslo_config import cfg +import oslo_messaging +import six + +from yardstick.common import messaging + + +LOG = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class NotificationHandler(object): + """Abstract class to define a endpoint object for a MessagingConsumer""" + + def __init__(self, id, ctx_pid, queue): + self._id = id + self._ctx_pid = ctx_pid + self._queue = queue + + +@six.add_metaclass(abc.ABCMeta) +class MessagingConsumer(object): + """Abstract class to implement a MQ consumer + + This abstract class allows a class implementing this interface to receive + the messages published by a `MessagingNotifier`. + """ + + def __init__(self, topic, pid, endpoints, fanout=True): + """Init function. + + :param topic: (string) MQ exchange topic + :param pid: (int) PID of the process implementing the MQ Notifier which + will be in the message context + :param endpoints: (list of class) list of classes implementing the + methods (see `MessagingNotifier.send_message) used by + the Notifier + :param fanout: (bool) MQ clients may request that a copy of the message + be delivered to all servers listening on a topic by + setting fanout to ``True``, rather than just one of them + :returns: `MessagingConsumer` class object + """ + + self._pid = pid + self._endpoints = endpoints + self._transport = oslo_messaging.get_rpc_transport( + cfg.CONF, url=messaging.TRANSPORT_URL) + self._target = oslo_messaging.Target(topic=topic, fanout=fanout, + server=messaging.SERVER) + self._server = oslo_messaging.get_rpc_server( + self._transport, self._target, self._endpoints, + executor=messaging.RPC_SERVER_EXECUTOR, + access_policy=oslo_messaging.DefaultRPCAccessPolicy) + + def start_rpc_server(self): + """Start the RPC server.""" + if self._server: + self._server.start() + + def stop_rpc_server(self): + """Stop the RPC server.""" + if self._server: + self._server.stop() + + def wait(self): + """Wait for message processing to complete (blocking).""" + if self._server: + self._server.wait() diff --git a/yardstick/common/messaging/payloads.py b/yardstick/common/messaging/payloads.py new file mode 100644 index 000000000..d29d79808 --- /dev/null +++ b/yardstick/common/messaging/payloads.py @@ -0,0 +1,53 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +import six + +from yardstick.common import exceptions + + +@six.add_metaclass(abc.ABCMeta) +class Payload(object): + """Base Payload class to transfer data through the MQ service""" + + REQUIRED_FIELDS = {'version'} + + def __init__(self, **kwargs): + """Init method + + :param kwargs: (dictionary) attributes and values of the object + :returns: Payload object + """ + + if not all(req_field in kwargs for req_field in self.REQUIRED_FIELDS): + _attrs = set(kwargs) - self.REQUIRED_FIELDS + missing_attributes = ', '.join(str(_attr) for _attr in _attrs) + raise exceptions.PayloadMissingAttributes( + missing_attributes=missing_attributes) + + for name, value in kwargs.items(): + setattr(self, name, value) + + self._fields = set(kwargs.keys()) + + def obj_to_dict(self): + """Returns a dictionary with the attributes of the object""" + return {field: getattr(self, field) for field in self._fields} + + @classmethod + def dict_to_obj(cls, _dict): + """Returns a Payload object built from the dictionary elements""" + return cls(**_dict) diff --git a/yardstick/common/messaging/producer.py b/yardstick/common/messaging/producer.py new file mode 100644 index 000000000..b6adc0c17 --- /dev/null +++ b/yardstick/common/messaging/producer.py @@ -0,0 +1,70 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import logging +import os + +from oslo_config import cfg +import oslo_messaging +import six + +from yardstick.common import messaging + + +LOG = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class MessagingProducer(object): + """Abstract class to implement a MQ producer + + This abstract class allows a class implementing this interface to publish + messages in a message queue. + """ + + def __init__(self, topic, pid=os.getpid(), fanout=True): + """Init function. + + :param topic: (string) MQ exchange topic + :param pid: (int) PID of the process implementing this MQ Notifier + :param fanout: (bool) MQ clients may request that a copy of the message + be delivered to all servers listening on a topic by + setting fanout to ``True``, rather than just one of them + :returns: `MessagingNotifier` class object + """ + self._topic = topic + self._pid = pid + self._fanout = fanout + self._transport = oslo_messaging.get_rpc_transport( + cfg.CONF, url=messaging.TRANSPORT_URL) + self._target = oslo_messaging.Target(topic=topic, fanout=fanout, + server=messaging.SERVER) + self._notifier = oslo_messaging.RPCClient(self._transport, + self._target) + + def send_message(self, method, payload): + """Send a cast message, that will invoke a method without blocking. + + The cast() method is used to invoke an RPC method that does not return + a value. cast() RPC requests may be broadcast to all Servers listening + on a given topic by setting the fanout Target property to ``True``. + + :param methos: (string) method name, that must be implemented in the + consumer endpoints + :param payload: (subclass `Payload`) payload content + """ + self._notifier.cast({'pid': self._pid}, + method, + **payload.obj_to_dict()) diff --git a/yardstick/common/openstack_utils.py b/yardstick/common/openstack_utils.py index 2785230c0..5208a2749 100644 --- a/yardstick/common/openstack_utils.py +++ b/yardstick/common/openstack_utils.py @@ -733,7 +733,7 @@ def create_security_group_full(neutron_client, sg_name, log.debug("Adding ICMP rules in security group '%s'...", sg_name) if not create_security_group_rule(neutron_client, sg_id, - 'ingress', 'icmp'): + 'ingress', 'icmp'): log.error("Failed to create the security group rule...") return None @@ -797,6 +797,18 @@ def delete_image(glance_client, image_id): # pragma: no cover return True +def list_images(shade_client=None): + if shade_client is None: + shade_client = get_shade_client() + + try: + return shade_client.list_images() + except exc.OpenStackCloudException as o_exc: + log.error("Error [list_images(shade_client)]." + "Exception message, '%s'", o_exc.orig_message) + return False + + # ********************************************* # CINDER # ********************************************* diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 357f66be8..44cc92a7c 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -136,6 +136,11 @@ def source_env(env_file): p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE, shell=True) output = p.communicate()[0] + + # sometimes output type would be binary_type, and it don't have splitlines + # method, so we need to decode + if isinstance(output, six.binary_type): + output = encodeutils.safe_decode(output) env = dict(line.split('=', 1) for line in output.splitlines() if '=' in line) os.environ.update(env) return env diff --git a/yardstick/tests/functional/common/messaging/__init__.py b/yardstick/tests/functional/common/messaging/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/functional/common/messaging/__init__.py diff --git a/yardstick/tests/functional/common/messaging/test_messaging.py b/yardstick/tests/functional/common/messaging/test_messaging.py new file mode 100644 index 000000000..96deeb35b --- /dev/null +++ b/yardstick/tests/functional/common/messaging/test_messaging.py @@ -0,0 +1,94 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +import os +import time + +from yardstick.common.messaging import consumer +from yardstick.common.messaging import payloads +from yardstick.common.messaging import producer +from yardstick.tests.functional import base + + +TOPIC = 'topic_MQ' +METHOD_INFO = 'info' + + +class DummyPayload(payloads.Payload): + REQUIRED_FIELDS = {'version', 'data'} + + +class DummyEndpoint(consumer.NotificationHandler): + + def info(self, ctxt, **kwargs): + if ctxt['pid'] == self._ctx_pid: + self._queue.put('ID {}, data: {}'.format(self._id, kwargs['data'])) + + +class DummyConsumer(consumer.MessagingConsumer): + + def __init__(self, id, ctx_pid, queue): + self._id = id + endpoints = [DummyEndpoint(id, ctx_pid, queue)] + super(DummyConsumer, self).__init__(TOPIC, ctx_pid, endpoints) + + +class DummyProducer(producer.MessagingProducer): + pass + + +def _run_consumer(id, ctx_pid, queue): + _consumer = DummyConsumer(id, ctx_pid, queue) + _consumer.start_rpc_server() + _consumer.wait() + + +class MessagingTestCase(base.BaseFunctionalTestCase): + + @staticmethod + def _terminate_consumers(num_consumers, processes): + for i in range(num_consumers): + processes[i].terminate() + + def test_run_five_consumers(self): + output_queue = multiprocessing.Queue() + num_consumers = 10 + ctx_id = os.getpid() + producer = DummyProducer(TOPIC, pid=ctx_id) + + processes = [] + for i in range(num_consumers): + processes.append(multiprocessing.Process( + name='consumer_{}'.format(i), + target=_run_consumer, + args=(i, ctx_id, output_queue))) + processes[i].start() + self.addCleanup(self._terminate_consumers, num_consumers, processes) + + time.sleep(2) # Let consumers to create the listeners + producer.send_message(METHOD_INFO, DummyPayload(version=1, + data='message 0')) + producer.send_message(METHOD_INFO, DummyPayload(version=1, + data='message 1')) + time.sleep(2) # Let consumers attend the calls + + output = [] + while not output_queue.empty(): + output.append(output_queue.get(True, 1)) + + self.assertEqual(num_consumers * 2, len(output)) + for i in range(num_consumers): + self.assertIn('ID {}, data: {}'.format(1, 'message 0'), output) + self.assertIn('ID {}, data: {}'.format(1, 'message 1'), output) diff --git a/yardstick/tests/unit/apiserver/resources/v2/__init__.py b/yardstick/tests/unit/apiserver/resources/v2/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/unit/apiserver/resources/v2/__init__.py diff --git a/yardstick/tests/unit/apiserver/resources/v2/test_images.py b/yardstick/tests/unit/apiserver/resources/v2/test_images.py new file mode 100644 index 000000000..ab131eec5 --- /dev/null +++ b/yardstick/tests/unit/apiserver/resources/v2/test_images.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import mock + +import unittest + +from yardstick.tests.unit.apiserver import APITestCase +from api.resources.v2.images import format_image_info + + +class V2ImagesTestCase(APITestCase): + @mock.patch('yardstick.common.openstack_utils.list_images') + @mock.patch('yardstick.common.utils.source_env') + def test_get(self, _, mock_list_images): + if self.app is None: + unittest.skip('host config error') + return + + single_image = mock.MagicMock() + single_image.name = 'yardstick-image' + single_image.size = 16384 + single_image.status = 'active' + single_image.updated_at = '2018-04-08' + + mock_list_images.return_value = [single_image] + url = 'api/v2/yardstick/images' + resp = self._get(url) + self.assertEqual(resp.get('status'), 1) + + +class FormatImageInfoTestCase(unittest.TestCase): + def test_format_image_info(self): + image = mock.MagicMock() + image.name = 'yardstick-image' + image.size = 1048576 + image.status = 'active' + image.updated_at = '2018-04-08' + + image_dict = format_image_info(image) + self.assertEqual(image_dict.get('size'), 1) diff --git a/yardstick/tests/unit/common/messaging/__init__.py b/yardstick/tests/unit/common/messaging/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/unit/common/messaging/__init__.py diff --git a/yardstick/tests/unit/common/messaging/test_consumer.py b/yardstick/tests/unit/common/messaging/test_consumer.py new file mode 100644 index 000000000..612dcaecd --- /dev/null +++ b/yardstick/tests/unit/common/messaging/test_consumer.py @@ -0,0 +1,54 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo_config import cfg +import oslo_messaging + +from yardstick.common import messaging +from yardstick.common.messaging import consumer +from yardstick.tests.unit import base as ut_base + + +class TestEndPoint(object): + def action_1(self): + pass + + +class _MessagingConsumer(consumer.MessagingConsumer): + pass + + +class MessagingConsumerTestCase(ut_base.BaseUnitTestCase): + + def test__init(self): + with mock.patch.object(oslo_messaging, 'get_rpc_server') as \ + mock_get_rpc_server, \ + mock.patch.object(oslo_messaging, 'get_rpc_transport') as \ + mock_get_rpc_transport, \ + mock.patch.object(oslo_messaging, 'Target') as \ + mock_Target: + mock_get_rpc_transport.return_value = 'test_rpc_transport' + mock_Target.return_value = 'test_Target' + + _MessagingConsumer('test_topic', 'test_pid', [TestEndPoint], + fanout=True) + mock_get_rpc_transport.assert_called_once_with( + cfg.CONF, url=messaging.TRANSPORT_URL) + mock_Target.assert_called_once_with( + topic='test_topic', fanout=True, server=messaging.SERVER) + mock_get_rpc_server.assert_called_once_with( + 'test_rpc_transport', 'test_Target', [TestEndPoint], + executor=messaging.RPC_SERVER_EXECUTOR, + access_policy=oslo_messaging.DefaultRPCAccessPolicy) diff --git a/yardstick/tests/unit/common/messaging/test_payloads.py b/yardstick/tests/unit/common/messaging/test_payloads.py new file mode 100644 index 000000000..00ec220c9 --- /dev/null +++ b/yardstick/tests/unit/common/messaging/test_payloads.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from yardstick.common import exceptions +from yardstick.common.messaging import payloads +from yardstick.tests.unit import base as ut_base + + +class _DummyPayload(payloads.Payload): + REQUIRED_FIELDS = {'version', 'key1', 'key2'} + + +class PayloadTestCase(ut_base.BaseUnitTestCase): + + def test__init(self): + payload = _DummyPayload(version=1, key1='value1', key2='value2') + self.assertEqual(1, payload.version) + self.assertEqual('value1', payload.key1) + self.assertEqual('value2', payload.key2) + self.assertEqual(3, len(payload._fields)) + + def test__init_missing_required_fields(self): + with self.assertRaises(exceptions.PayloadMissingAttributes): + _DummyPayload(key1='value1', key2='value2') + + def test_obj_to_dict(self): + payload = _DummyPayload(version=1, key1='value1', key2='value2') + payload_dict = payload.obj_to_dict() + self.assertEqual({'version': 1, 'key1': 'value1', 'key2': 'value2'}, + payload_dict) + + def test_dict_to_obj(self): + _dict = {'version': 2, 'key1': 'value100', 'key2': 'value200'} + payload = _DummyPayload.dict_to_obj(_dict) + self.assertEqual(set(_dict.keys()), payload._fields) diff --git a/yardstick/tests/unit/common/messaging/test_producer.py b/yardstick/tests/unit/common/messaging/test_producer.py new file mode 100644 index 000000000..0289689dc --- /dev/null +++ b/yardstick/tests/unit/common/messaging/test_producer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo_config import cfg +import oslo_messaging + +from yardstick.common import messaging +from yardstick.common.messaging import producer +from yardstick.tests.unit import base as ut_base + + +class _MessagingProducer(producer.MessagingProducer): + pass + + +class MessagingProducerTestCase(ut_base.BaseUnitTestCase): + + def test__init(self): + with mock.patch.object(oslo_messaging, 'RPCClient') as \ + mock_RPCClient, \ + mock.patch.object(oslo_messaging, 'get_rpc_transport') as \ + mock_get_rpc_transport, \ + mock.patch.object(oslo_messaging, 'Target') as \ + mock_Target: + mock_get_rpc_transport.return_value = 'test_rpc_transport' + mock_Target.return_value = 'test_Target' + + _MessagingProducer('test_topic', 'test_pid', fanout=True) + mock_get_rpc_transport.assert_called_once_with( + cfg.CONF, url=messaging.TRANSPORT_URL) + mock_Target.assert_called_once_with( + topic='test_topic', fanout=True, server=messaging.SERVER) + mock_RPCClient.assert_called_once_with('test_rpc_transport', + 'test_Target') diff --git a/yardstick/tests/unit/common/test_openstack_utils.py b/yardstick/tests/unit/common/test_openstack_utils.py index 3b7e8eaa1..5c7e5bfab 100644 --- a/yardstick/tests/unit/common/test_openstack_utils.py +++ b/yardstick/tests/unit/common/test_openstack_utils.py @@ -264,3 +264,21 @@ class CreateSecurityGroupRuleTestCase(unittest.TestCase): self.mock_shade_client, self.secgroup_name_or_id) mock_logger.error.assert_called_once() self.assertFalse(output) + + +class ListImageTestCase(unittest.TestCase): + + def test_list_images(self): + mock_shade_client = mock.MagicMock() + mock_shade_client.list_images.return_value = [] + openstack_utils.list_images(mock_shade_client) + + @mock.patch.object(openstack_utils, 'log') + def test_list_images_exception(self, mock_logger): + mock_shade_client = mock.MagicMock() + mock_shade_client.list_images = mock.MagicMock() + mock_shade_client.list_images.side_effect = ( + exc.OpenStackCloudException('error message')) + images = openstack_utils.list_images(mock_shade_client) + mock_logger.error.assert_called_once() + self.assertFalse(images) |