diff options
90 files changed, 2409 insertions, 1332 deletions
diff --git a/ansible/build_yardstick_image.yml b/ansible/build_yardstick_image.yml index 429ab88aa..072c12c66 100644 --- a/ansible/build_yardstick_image.yml +++ b/ansible/build_yardstick_image.yml @@ -75,18 +75,18 @@ ignore_errors: true - name: Debug dump loop devices - command: losetup - register: losetup_output - - - debug: - var: losetup_output - verbosity: 2 + command: losetup -a + ignore_errors: true - name: delete loop devices for image file # use this because kpartx -dv will fail if raw_imgfile was delete # but in theory we could have deleted file still attached to loopback device? # use grep because of // and awk - shell: losetup -O NAME,BACK-FILE | grep "{{ raw_imgfile_basename }}" | awk '{ print $1 }' | xargs -l1 losetup -d + shell: losetup -O NAME,BACK-FILE | grep "{{ raw_imgfile_basename }}" | awk '{ print $1 }' | xargs -l1 losetup -v -d + ignore_errors: true + + - name: Debug dump loop devices again + command: losetup -a ignore_errors: true - name: delete {{ raw_imgfile }} diff --git a/ansible/infra_deploy.yml b/ansible/infra_deploy.yml index 4ad21af00..029879502 100644 --- a/ansible/infra_deploy.yml +++ b/ansible/infra_deploy.yml @@ -13,9 +13,12 @@ # limitations under the License. --- - hosts: jumphost + vars: + rs_file: "{{ RS_FILE }}" + clean_up: "{{ CLEAN_UP | default(False) }}" # If True will be delete all VMs, networks, disk images roles: - - infra_check_requirements - infra_destroy_previous_configuration + - infra_check_requirements - infra_create_network - infra_create_vms diff --git a/ansible/roles/docker/tasks/Debian.yml b/ansible/roles/docker/tasks/Debian.yml index cf4128774..7f998de45 100644 --- a/ansible/roles/docker/tasks/Debian.yml +++ b/ansible/roles/docker/tasks/Debian.yml @@ -12,15 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - - name: add Ubuntu docker repo - apt_repository: repo='deb [trusted=yes] {{ ubuntu_docker_url }} ubuntu-{{ ansible_distribution_release }} main' state=present - - - name: ensure correct docker version - action: "{{ ansible_pkg_mgr }} name={{ item }} state=present force=yes" - with_items: "{{ docker_packages[ansible_os_family] }}" - - - name: remove Ubuntu docker repo - apt_repository: - repo: 'deb [trusted=yes] {{ ubuntu_docker_url }} ubuntu-{{ ansible_distribution_release }} main' - state: absent - update_cache: no + - name: Install docker.io + action: "{{ ansible_pkg_mgr }} name=docker.io state=present force=yes" diff --git a/ansible/roles/docker/vars/main.yml b/ansible/roles/docker/vars/main.yml index 8b5077490..a735d523d 100644 --- a/ansible/roles/docker/vars/main.yml +++ b/ansible/roles/docker/vars/main.yml @@ -16,5 +16,3 @@ docker_project_url: https://yum.dockerproject.org docker_packages: "RedHat": - docker-engine-1.13.1 - "Debian": - - docker-engine=1.13.1* diff --git a/ansible/roles/download_collectd/defaults/main.yml b/ansible/roles/download_collectd/defaults/main.yml index 9beaeb897..3f5632394 100644 --- a/ansible/roles/download_collectd/defaults/main.yml +++ b/ansible/roles/download_collectd/defaults/main.yml @@ -1,4 +1,4 @@ --- collectd_url: "https://github.com/collectd/collectd.git" collectd_dest: "{{ clone_dest }}/collectd" -collectd_version: "c870991a2d614e51c03c0da76e9aef997343551b" +collectd_version: "collectd-5.8" diff --git a/ansible/roles/download_dpdk/defaults/main.yml b/ansible/roles/download_dpdk/defaults/main.yml index fb42ed48a..d548280f5 100644 --- a/ansible/roles/download_dpdk/defaults/main.yml +++ b/ansible/roles/download_dpdk/defaults/main.yml @@ -1,10 +1,14 @@ --- dpdk_version: "17.02" -dpdk_url: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.zip" +dpdk_url: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz" dpdk_file: "{{ dpdk_url|basename }}" -dpdk_unarchive: "{{ dpdk_file|regex_replace('[.]zip$', '') }}" +dpdk_unarchive: "{{ dpdk_file|regex_replace('[.]tar[.]gz$', '') }}" dpdk_dest: "{{ clone_dest }}/" +#Note DPDK 17.08 17.11 and 18.02 are currently unsupported due to prox build issues dpdk_sha256s: - "17.02": "sha256:c675285d5eab8e7e8537a10bdae6f03c603caa80cb4bf5b055ddd482e3b7d67f" - "16.04": "sha256:65e587af439b21abf54c644f86ba5ce7ea65657b3272480dcd78b769d2aecac3" - "16.07": "sha256:46fee52f9b3ff54df869414c69f122aea8f744de5ed5f8fb275a22cc9cefe094" + "16.07": "sha256:d876e4b2a7101f28e7e345d3c88e66afe877d15f0159c19c5bc5bc26b7b7d788" + "17.02": "sha256:b07b546e910095174bdb6152bb0d7ce057cc4b79aaa74771aeee4e8a7219fb38" + "17.05": "sha256:763bfb7e1765efcc949e79d645dc9f1ebd16591431ba0db5ce22becd928dcd0a" + "17.08": "sha256:3a08addbff45c636538514e9a5838fb91ea557661a4c071e03a9a6987d46e5b6" #unsupported + "17.11": "sha256:77a727bb3834549985f291409c9a77a1e8be1c9329ce4c3eb19a22d1461022e4" #unsupported + "18.02": "sha256:f1210310fd5f01a3babe3a09d9b3e5a9db791c2ec6ecfbf94ade9f893a0632b8" #unsupported diff --git a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml index 314ee30af..5e616335a 100644 --- a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml +++ b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml @@ -14,18 +14,18 @@ --- - name: Destroy old networks created by virt virt_net: - name: "{{ network_item.name }}" + name: "{{ network_item }}" command: destroy - when: network_item.name in virt_nets.list_nets + when: clean_up | bool or network_item in deploy_nets -# Ignoring erros as network can be created without being defined. +# Ignoring errors as network can be created without being defined. # This can happen if a user manually creates a network using the virsh command. # If the network is not defined the undefine code will throw an error. - name: Undefine old networks defined by virt virt_net: - name: "{{ network_item.name }}" + name: "{{ network_item }}" command: undefine - when: network_item.name in virt_nets.list_nets + when: clean_up | bool or network_item in deploy_nets ignore_errors: yes - name: Check if "ovs-vsctl" command is present @@ -34,15 +34,20 @@ ignore_errors: yes - name: Destroy OVS bridge if it exists - command: ovs-vsctl --if-exists -- del-br "{{ network_item.name }}" - when: ovs_vsctl_present.rc == 0 + command: ovs-vsctl --if-exists -- del-br "{{ network_item }}" + when: + - ovs_vsctl_present.rc == 0 + - clean_up | bool or network_item in deploy_nets + ignore_errors: yes - name: Check if linux bridge is present - stat: path="{{ '/sys/class/net/'+network_item.name+'/brif/' }}" + stat: path="{{ '/sys/class/net/' + network_item + '/brif/' }}" register: check_linux_bridge - name: Remove linux bridge if it exists shell: | - ifconfig "{{ network_item.name }}" down - brctl delbr "{{ network_item.name }}" - when: check_linux_bridge.stat.exists + ifconfig "{{ network_item }}" down + brctl delbr "{{ network_item }}" + when: + - check_linux_bridge.stat.exists + - clean_up | bool or network_item in deploy_nets diff --git a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml index 5e43ee81e..91e949344 100644 --- a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml +++ b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml @@ -16,14 +16,14 @@ - name: Destroy old VMs virt: command: destroy - name: "{{ node_item.hostname }}" - when: node_item.hostname in virt_vms.list_vms + name: "{{ vmhost_item }}" + when: clean_up | bool or vmhost_item in deploy_vms ignore_errors: yes # Ignore errors as VM can be running while undefined - name: Undefine old VMs virt: command: undefine - name: "{{ node_item.hostname }}" - when: node_item.hostname in virt_vms.list_vms + name: "{{ vmhost_item }}" + when: clean_up | bool or vmhost_item in deploy_vms ignore_errors: yes diff --git a/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml b/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml index e6c2c0229..6c4aa33cf 100644 --- a/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml +++ b/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -- name: Include +- name: Include input file include_vars: file: "{{ rs_file }}" name: infra_deploy_vars @@ -25,16 +25,40 @@ virt: command=list_vms register: virt_vms +- set_fact: + deploy_vms: "{{ deploy_vms | default([]) + [item.hostname] }}" + with_items: "{{ infra_deploy_vars.nodes }}" + +- name: Define old disk images to delete + shell: virsh domblklist {{ item }} | awk '/\/.*/ { print $2 }' + when: clean_up | bool or item in deploy_vms + with_items: "{{ virt_vms.list_vms }}" + register: virt_img + +- set_fact: + images: "{{ images | default([]) + item.stdout_lines }}" + when: item.stdout_lines is defined and item.stdout_lines | length > 0 + with_items: "{{ virt_img.results }}" + - name: Destroy old VMs include_tasks: delete_vm.yml - extra_vars: "{{ virt_vms }}" loop_control: - loop_var: node_item - with_items: "{{ infra_deploy_vars.nodes }}" + loop_var: vmhost_item + with_items: "{{ virt_vms.list_vms }}" + +- set_fact: + deploy_nets: "{{ deploy_nets | default([]) + [item.name] }}" + with_items: "{{ infra_deploy_vars.networks }}" - name: Delete old networks include_tasks: delete_network.yml - extra_vars: "{{ virt_nets }}" loop_control: loop_var: network_item - with_items: "{{ infra_deploy_vars.networks }}" + with_items: "{{ virt_nets.list_nets }}" + +- name: Delete old disk images + file: + path: "{{ item }}" + state: absent + when: images is defined and images | length > 0 + with_items: "{{ images }}" diff --git a/ansible/roles/install_civetweb/defaults/main.yml b/ansible/roles/install_civetweb/defaults/main.yml index 2235fe172..ed5ab27f2 100644 --- a/ansible/roles/install_civetweb/defaults/main.yml +++ b/ansible/roles/install_civetweb/defaults/main.yml @@ -15,6 +15,7 @@ civetweb_dest: "{{ clone_dest }}/civetweb" civetweb_build_dependencies: Debian: + - libjson-c-dev=0.11-4ubuntu2 - libjson0 - libjson0-dev - libssl-dev diff --git a/ansible/roles/install_dpdk/tasks/main.yml b/ansible/roles/install_dpdk/tasks/main.yml index e82ad8363..5bcfb50b1 100644 --- a/ansible/roles/install_dpdk/tasks/main.yml +++ b/ansible/roles/install_dpdk/tasks/main.yml @@ -116,7 +116,7 @@ - name: copy dpdk-devbind.py to correct location copy: - src: "{{ dpdk_devbind[dpdk_version] }}" + src: "{{ dpdk_devbind_usertools if dpdk_version|float >= 17.02 else dpdk_devbind_tools }}" dest: "{{ INSTALL_BIN_PATH }}/dpdk-devbind.py" remote_src: yes force: yes diff --git a/ansible/roles/install_dpdk/vars/main.yml b/ansible/roles/install_dpdk/vars/main.yml index 45bcc339b..957f47e99 100644 --- a/ansible/roles/install_dpdk/vars/main.yml +++ b/ansible/roles/install_dpdk/vars/main.yml @@ -1,9 +1,10 @@ --- -dpdk_make_arch: x86_64-native-linuxapp-gcc +dpdk_make_archs: + "amd64": "x86_64-native-linuxapp-gcc" + "arm64": "arm64-native-linuxapp-gcc" +dpdk_make_arch: "{{ dpdk_make_archs[YARD_IMG_ARCH] }}" dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra" hugetable_mount: /mnt/huge -dpdk_devbind: - "16.07": "{{ dpdk_path }}/tools/dpdk-devbind.py" - "17.02": "{{ dpdk_path }}/usertools/dpdk-devbind.py" - "17.04": "{{ dpdk_path }}/usertools/dpdk-devbind.py" +dpdk_devbind_tools: "{{ dpdk_path }}/tools/dpdk-devbind.py" +dpdk_devbind_usertools: "{{ dpdk_path }}/usertools/dpdk-devbind.py" dpdk_pmd_path: /usr/lib/dpdk-pmd/ diff --git a/ansible/roles/install_dpdk_shared/tasks/Debian.yml b/ansible/roles/install_dpdk_shared/tasks/Debian.yml index c77e4f96a..641d8f9cd 100755 --- a/ansible/roles/install_dpdk_shared/tasks/Debian.yml +++ b/ansible/roles/install_dpdk_shared/tasks/Debian.yml @@ -16,6 +16,7 @@ action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" with_items: - libpcap-dev + - libnuma-dev - name: Install kernel headers action: "{{ ansible_pkg_mgr }} name=linux-headers-{{ dpdk_kernel }} state=present" diff --git a/ansible/roles/install_dpdk_shared/tasks/main.yml b/ansible/roles/install_dpdk_shared/tasks/main.yml index 65954be04..94b9215bd 100644 --- a/ansible/roles/install_dpdk_shared/tasks/main.yml +++ b/ansible/roles/install_dpdk_shared/tasks/main.yml @@ -42,6 +42,15 @@ set_fact: RTE_KERNELDIR: "/lib/modules/{{ dpdk_kernel }}/build" +# make clean must be run here as DPDK-shared is a copy of the DPDK directory +# which has already run the make command +# no T= target for clean +- command: make -j {{ ansible_processor_vcpus }} clean O={{ dpdk_make_arch }} + args: + chdir: "{{ dpdk_shared_path }}" + environment: + RTE_KERNELDIR: "{{ RTE_KERNELDIR }}" + - command: make -j {{ ansible_processor_vcpus }} config T={{ dpdk_make_arch }} O={{ dpdk_make_arch }} args: chdir: "{{ dpdk_shared_path }}" @@ -85,13 +94,6 @@ regexp: '^CONFIG_RTE_EAL_PMD_PATH=""' line: 'CONFIG_RTE_EAL_PMD_PATH="{{ dpdk_pmd_path }}"' - # no T= target for clean -- command: make -j {{ ansible_processor_vcpus }} clean O={{ dpdk_make_arch }} - args: - chdir: "{{ dpdk_shared_path }}" - environment: - RTE_KERNELDIR: "{{ RTE_KERNELDIR }}" - # TODO: disable ASLR - command: make -j {{ ansible_processor_vcpus }} diff --git a/ansible/roles/install_dpdk_shared/vars/main.yml b/ansible/roles/install_dpdk_shared/vars/main.yml index 45bcc339b..b663cedd2 100644 --- a/ansible/roles/install_dpdk_shared/vars/main.yml +++ b/ansible/roles/install_dpdk_shared/vars/main.yml @@ -1,9 +1,8 @@ --- -dpdk_make_arch: x86_64-native-linuxapp-gcc +dpdk_make_archs: + "amd64": "x86_64-native-linuxapp-gcc" + "arm64": "arm64-native-linuxapp-gcc" +dpdk_make_arch: "{{ dpdk_make_archs[YARD_IMG_ARCH] }}" dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra" hugetable_mount: /mnt/huge -dpdk_devbind: - "16.07": "{{ dpdk_path }}/tools/dpdk-devbind.py" - "17.02": "{{ dpdk_path }}/usertools/dpdk-devbind.py" - "17.04": "{{ dpdk_path }}/usertools/dpdk-devbind.py" dpdk_pmd_path: /usr/lib/dpdk-pmd/ diff --git a/api/resources/v2/images.py b/api/resources/v2/images.py index 0c36a0a26..c3e5ee73e 100644 --- a/api/resources/v2/images.py +++ b/api/resources/v2/images.py @@ -18,8 +18,7 @@ from api.database.v2.handlers import V2ImageHandler from api.database.v2.handlers import V2EnvironmentHandler from yardstick.common.utils import result_handler from yardstick.common.utils import source_env -from yardstick.common.utils import change_obj_to_dict -from yardstick.common.openstack_utils import get_nova_client +from yardstick.common import openstack_utils from yardstick.common.openstack_utils import get_glance_client from yardstick.common import constants as consts @@ -47,39 +46,21 @@ class V2Images(ApiResource): def get(self): try: source_env(consts.OPENRC) - except Exception: + except OSError: return result_handler(consts.API_ERROR, 'source openrc error') - nova_client = get_nova_client() - try: - images_list = nova_client.images.list() - except Exception: + image_list = openstack_utils.list_images() + + if image_list is False: return result_handler(consts.API_ERROR, 'get images error') - else: - images = {i.name: self.get_info(change_obj_to_dict(i)) for i in images_list} + + images = {i.name: format_image_info(i) for i in image_list} return result_handler(consts.API_SUCCESS, {'status': 1, 'images': images}) def post(self): return self._dispatch_post() - def get_info(self, data): - try: - size = data['OS-EXT-IMG-SIZE:size'] - except KeyError: - size = None - else: - size = float(size) / 1024 / 1024 - - result = { - 'name': data.get('name', ''), - 'discription': data.get('description', ''), - 'size': size, - 'status': data.get('status'), - 'time': data.get('updated') - } - return result - def load_image(self, args): try: image_name = args['name'] @@ -268,7 +249,7 @@ class V2Images(ApiResource): r = requests.head(url) try: file_size = int(r.headers['content-length']) - except Exception: + except (TypeError, ValueError): return with open(path, 'wb') as f: @@ -303,14 +284,13 @@ class V2Image(ApiResource): except ValueError: return result_handler(consts.API_ERROR, 'no such image id') - nova_client = get_nova_client() - images = nova_client.images.list() + images = openstack_utils.list_images() try: image = next((i for i in images if i.name == image.name)) except StopIteration: pass - return_image = self.get_info(change_obj_to_dict(image)) + return_image = format_image_info(image) return_image['id'] = image_id return result_handler(consts.API_SUCCESS, {'image': return_image}) @@ -349,19 +329,16 @@ class V2Image(ApiResource): return result_handler(consts.API_SUCCESS, {'image': image_id}) - def get_info(self, data): - try: - size = data['OS-EXT-IMG-SIZE:size'] - except KeyError: - size = None - else: - size = float(size) / 1024 / 1024 - - result = { - 'name': data.get('name', ''), - 'description': data.get('description', ''), - 'size': size, - 'status': data.get('status'), - 'time': data.get('updated') - } - return result + +def format_image_info(image): + image_dict = {} + + if image is None: + return image_dict + + image_dict['name'] = image.name + image_dict['size'] = float(image.size) / 1024 / 1024 + image_dict['status'] = image.status.upper() + image_dict['time'] = image.updated_at + + return image_dict diff --git a/docker/Dockerfile.aarch64.patch b/docker/Dockerfile.aarch64.patch index de272f5a8..e8dbea288 100644 --- a/docker/Dockerfile.aarch64.patch +++ b/docker/Dockerfile.aarch64.patch @@ -42,8 +42,8 @@ index 2ee5b4c..23e5ea5 100644 RUN pip install appdirs==1.4.0 pyopenssl==17.5.0 python-openstackclient==3.11.0 python-heatclient==1.11.0 @@ -43,8 +44,8 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf -# nginx=5000, rabbitmq=5672 -EXPOSE 5000 5672 + # nginx=5000, rabbitmq=5672 + EXPOSE 5000 5672 -ADD http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img ${IMAGE_DIR} -ADD http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img ${IMAGE_DIR} diff --git a/docker/supervisor.sh b/docker/supervisor.sh index b67de2212..44e34fb38 100755 --- a/docker/supervisor.sh +++ b/docker/supervisor.sh @@ -11,7 +11,7 @@ # nginx service start when boot supervisor_config='/etc/supervisor/conf.d/yardstick.conf' -if [[ ! -e "${supervisor_config}" ]];then +if [[ ! -e "${supervisor_config}" ]]; then cat << EOF > "${supervisor_config}" [supervisord] nodaemon = true @@ -22,5 +22,8 @@ command = service nginx restart [program:yardstick_uwsgi] directory = /etc/yardstick command = uwsgi -i yardstick.ini + +[program:rabbitmq] +command = service rabbitmq-server restart EOF fi diff --git a/docs/release/release-notes/release-notes.rst b/docs/release/release-notes/release-notes.rst index 4ebf0eceb..6598a2751 100644 --- a/docs/release/release-notes/release-notes.rst +++ b/docs/release/release-notes/release-notes.rst @@ -1,7 +1,8 @@ +======= License ======= -OPNFV Euphrates release note for Yardstick Docs +OPNFV Fraser release note for Yardstick Docs are licensed under a Creative Commons Attribution 4.0 International License. You should have received a copy of the license along with this. If not, see <http://creativecommons.org/licenses/by/4.0/>. @@ -9,8 +10,9 @@ If not, see <http://creativecommons.org/licenses/by/4.0/>. The *Yardstick framework*, the *Yardstick test cases* are open-source software, licensed under the terms of the Apache License, Version 2.0. -OPNFV Euphrates Release Note for Yardstick -========================================== +======================================= +OPNFV Fraser Release Note for Yardstick +======================================= .. toctree:: :maxdepth: 2 @@ -23,50 +25,43 @@ OPNFV Euphrates Release Note for Yardstick Abstract --------- +======== This document describes the release note of Yardstick project. Version History ---------------- +=============== +-------------------+-----------+---------------------------------+ | *Date* | *Version* | *Comment* | | | | | +-------------------+-----------+---------------------------------+ -| December 15, 2017 | 5.1.0 | Yardstick for Euphrates release | -| | | | -+-------------------+-----------+---------------------------------+ -| October 20, 2017 | 5.0.0 | Yardstick for Euphrates release | +| April 27, 2018 | 6.0.0 | Yardstick for Fraser release | | | | | +-------------------+-----------+---------------------------------+ Important Notes ---------------- +=============== The software delivered in the OPNFV Yardstick_ Project, comprising the -*Yardstick framework*, the *Yardstick test cases* and the experimental -framework *Apex Lake* is a realization of the methodology in ETSI-ISG -NFV-TST001_. +*Yardstick framework*, and the *Yardstick test cases* is a realization of +the methodology in ETSI-ISG NFV-TST001_. The *Yardstick* framework is *installer*, *infrastructure* and *application* independent. -OPNFV Euphrates Release ------------------------ +OPNFV Fraser Release +==================== -This Euphrates release provides *Yardstick* as a framework for NFVI testing +This Fraser release provides *Yardstick* as a framework for NFVI testing and OPNFV feature testing, automated in the OPNFV CI pipeline, including: * Documentation generated with Sphinx * User Guide - * Developer Guide - * Release notes (this document) - * Results * Automated Yardstick test suite (daily, weekly) @@ -84,39 +79,29 @@ and OPNFV feature testing, automated in the OPNFV CI pipeline, including: * Yardstick plug-in configuration yaml files, plug-in install/remove scripts -For Euphrates release, the *Yardstick framework* is used for the following +For Fraser release, the *Yardstick framework* is used for the following testing: * OPNFV platform testing - generic test cases to measure the categories: * Compute - * Network - * Storage -* OPNFV platform network service benchmarking(NSB) +* OPNFV platform network service benchmarking (NSB) * NSB * Test cases for the following OPNFV Projects: * Container4NFV - * High Availability - * IPv6 - * KVM - * Parser - * StorPerf - * VSperf - * virtual Traffic Classifier - The *Yardstick framework* is developed in the OPNFV community, by the Yardstick_ team. @@ -126,49 +111,47 @@ Yardstick_ team. Release Data ------------- +============ +--------------------------------+-----------------------+ | **Project** | Yardstick | | | | +--------------------------------+-----------------------+ -| **Repo/tag** | yardstick/opnfv-5.1.0 | +| **Repo/tag** | yardstick/opnfv-6.0.0 | | | | +--------------------------------+-----------------------+ -| **Yardstick Docker image tag** | opnfv-5.1.0 | +| **Yardstick Docker image tag** | opnfv-6.0.0 | | | | +--------------------------------+-----------------------+ -| **Release designation** | Euphrates | +| **Release designation** | Fraser | | | | +--------------------------------+-----------------------+ -| **Release date** | December 15, 2017 | +| **Release date** | April 27, 2018 | | | | +--------------------------------+-----------------------+ -| **Purpose of the delivery** | OPNFV Euphrates 5.1.0 | +| **Purpose of the delivery** | OPNFV Fraser 6.0.0 | | | | +--------------------------------+-----------------------+ Deliverables ------------- +============ Documents -^^^^^^^^^ +--------- - - User Guide: http://docs.opnfv.org/en/stable-euphrates/submodules/yardstick/docs/testing/user/userguide/index.html + - User Guide: http://docs.opnfv.org/en/stable-fraser/submodules/yardstick/docs/testing/user/userguide/index.html - - Developer Guide: http://docs.opnfv.org/en/stable-euphrates/submodules/yardstick/docs/testing/developer/devguide/index.html + - Developer Guide: http://docs.opnfv.org/en/stable-fraser/submodules/yardstick/docs/testing/developer/devguide/index.html Software Deliverables -^^^^^^^^^^^^^^^^^^^^^ - +--------------------- - - The Yardstick Docker image: https://hub.docker.com/r/opnfv/yardstick (tag: opnfv-5.1.0) + - The Yardstick Docker image: https://hub.docker.com/r/opnfv/yardstick (tag: opnfv-6.0.0) - -New Contexts -############ +List of Contexts +^^^^^^^^^^^^^^^^ +--------------+-------------------------------------------+ | **Context** | **Description** | @@ -188,31 +171,40 @@ New Contexts +--------------+-------------------------------------------+ -New Runners -########### - -+--------------+-------------------------------------------------------+ -| **Runner** | **Description** | -| | | -+--------------+-------------------------------------------------------+ -| *Arithmetic* | Steps every run arithmetically according to specified | -| | input value | -| | | -+--------------+-------------------------------------------------------+ -| *Duration* | Runs for a specified period of time | -| | | -+--------------+-------------------------------------------------------+ -| *Iteration* | Runs for a specified number of iterations | -| | | -+--------------+-------------------------------------------------------+ -| *Sequence* | Selects input value to a scenario from an input file | -| | and runs all entries sequentially | -| | | -+--------------+-------------------------------------------------------+ - - -New Scenarios -############# +List of Runners +^^^^^^^^^^^^^^^ + +Note: Yardstick Fraser 6.0.0 add two new Runners, "Dynamictp" and "Search". + ++---------------+-------------------------------------------------------+ +| **Runner** | **Description** | +| | | ++---------------+-------------------------------------------------------+ +| *Arithmetic* | Steps every run arithmetically according to specified | +| | input value | +| | | ++---------------+-------------------------------------------------------+ +| *Duration* | Runs for a specified period of time | +| | | ++---------------+-------------------------------------------------------+ +| *Iteration* | Runs for a specified number of iterations | +| | | ++---------------+-------------------------------------------------------+ +| *Sequence* | Selects input value to a scenario from an input file | +| | and runs all entries sequentially | +| | | ++---------------+-------------------------------------------------------+ +| **Dynamictp** | A runner that searches for the max throughput with | +| | binary search | +| | | ++---------------+-------------------------------------------------------+ +| **Search** | A runner that runs a specific time before it returns | +| | | ++---------------+-------------------------------------------------------+ + + +List of Scenarios +^^^^^^^^^^^^^^^^^ +----------------+-----------------------------------------------------+ | **Category** | **Delivered** | @@ -234,224 +226,138 @@ New Scenarios | | | +----------------+-----------------------------------------------------+ | *Compute* | * cpuload | -| | | | | * cyclictest | -| | | | | * lmbench | -| | | | | * lmbench_cache | -| | | | | * perf | -| | | | | * unixbench | -| | | | | * ramspeed | -| | | | | * cachestat | -| | | | | * memeoryload | -| | | | | * computecapacity | -| | | | | * SpecCPU2006 | | | | +----------------+-----------------------------------------------------+ | *Networking* | * iperf3 | -| | | | | * netperf | -| | | | | * netperf_node | -| | | | | * ping | -| | | | | * ping6 | -| | | | | * pktgen | -| | | | | * sfc | -| | | | | * sfc with tacker | -| | | -| | * vtc instantion validation | -| | | -| | * vtc instantion validation with noisy neighbors | -| | | -| | * vtc throughput | -| | | -| | * vtc throughput in the presence of noisy neighbors | -| | | | | * networkcapacity | -| | | | | * netutilization | -| | | | | * nstat | -| | | | | * pktgenDPDK | | | | +----------------+-----------------------------------------------------+ | *Parser* | Tosca2Heat | | | | +----------------+-----------------------------------------------------+ -| *Storage* | fio | -| | | -| | bonnie++ | -| | | -| | storagecapacity | +| *Storage* | * fio | +| | * bonnie++ | +| | * storagecapacity | | | | +----------------+-----------------------------------------------------+ | *StorPerf* | storperf | | | | +----------------+-----------------------------------------------------+ -| *NSB* | vPE thoughput test case | +| *NSB* | vFW thoughput test case | | | | +----------------+-----------------------------------------------------+ - New Test cases -^^^^^^^^^^^^^^ +-------------- * Generic NFVI test cases - * OPNFV_YARDSTICK_TCO78 - SPEC CPU 2006 - - * OPNFV_YARDSTICK_TCO79 - Bonnie++ - -* Kubernetes Test cases + * OPNFV_YARDSTICK_TCO84 - SPEC CPU 2006 for VM - * OPNFV_YARDSTICK_TCO80 - NETWORK LATENCY BETWEEN CONTAINER +* HA Test cases - * OPNFV_YARDSTICK_TCO81 - NETWORK LATENCY BETWEEN CONTAINER AND VM + * OPNFV_YARDSTICK_TC087 - SDN Controller resilience in non-HA configuration + * OPNFV_YARDSTICK_TC090 - Control node Openstack service down - database instance + * OPNFV_YARDSTICK_TC091 - Control node Openstack service down - heat-api Version Change --------------- +============== Module Version Changes -^^^^^^^^^^^^^^^^^^^^^^ +---------------------- -This is the fifth tracked release of Yardstick. It is based on following +This is the sixth tracked release of Yardstick. It is based on following upstream versions: -- OpenStack Ocata - -- OpenDayLight Nitrogen - -- ONOS Junco +- OpenStack Pike +- OpenDayLight Oxygen Document Version Changes -^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------ -This is the fifth tracked version of the Yardstick framework in OPNFV. +This is the sixth tracked version of the Yardstick framework in OPNFV. It includes the following documentation updates: - Yardstick User Guide: add "network service benchmarking(NSB)" chapter; add "Yardstick - NSB Testing -Installation" chapter; add "Yardstick API" chapter; add "Yardstick user interface" chapter; Update Yardstick installation chapter; - - Yardstick Developer Guide - - Yardstick Release Notes for Yardstick: this document Feature additions -^^^^^^^^^^^^^^^^^ - -- Yardstick RESTful API support - -- Network service benchmarking - -- Stress testing with Bottlenecks team - -- Yardstick framework improvement: - - - yardstick report CLI - - - Node context support OpenStack configuration via Ansible - - - Https support +----------------- - - Kubernetes context type - -- Yardstick container local GUI - -- Python 3 support +- Plugin-based test cases support Heat context +- SR-IOV support for the Heat context +- Support using existing network in Heat context +- Support running test cases with existing VNFs/without destroying VNF in Heat context +- Add vFW scale-up template +- Improvements of unit tests and gating +- GUI improvement about passing parameters Scenario Matrix ---------------- - -For Euphrates 5.0.0, Yardstick was tested on the following scenarios: - -+--------------------------+------+---------+------+------+ -| Scenario | Apex | Compass | Fuel | Joid | -+==========================+======+=========+======+======+ -| os-nosdn-nofeature-noha | | | X | X | -+--------------------------+------+---------+------+------+ -| os-nosdn-nofeature-ha | X | X | X | X | -+--------------------------+------+---------+------+------+ -| os-odl_l2-nofeature-ha | | X | X | X | -+--------------------------+------+---------+------+------+ -| os-odl_l2-nofeature-noha | | | X | | -+--------------------------+------+---------+------+------+ -| os-odl_l3-nofeature-ha | X | X | X | | -+--------------------------+------+---------+------+------+ -| os-odl_l3-nofeature-noha | | | X | | -+--------------------------+------+---------+------+------+ -| os-onos-sfc-ha | | | | | -+--------------------------+------+---------+------+------+ -| os-onos-nofeature-ha | | X | | X | -+--------------------------+------+---------+------+------+ -| os-onos-nofeature-noha | | | | | -+--------------------------+------+---------+------+------+ -| os-odl_l2-sfc-ha | | | X | | -+--------------------------+------+---------+------+------+ -| os-odl_l2-sfc-noha | | | X | | -+--------------------------+------+---------+------+------+ -| os-odl_l2-bgpvpn-ha | X | | X | | -+--------------------------+------+---------+------+------+ -| os-odl_l2-bgpvpn-noha | | | X | | -+--------------------------+------+---------+------+------+ -| os-nosdn-kvm-ha | X | | X | | -+--------------------------+------+---------+------+------+ -| os-nosdn-kvm-noha | | | X | | -+--------------------------+------+---------+------+------+ -| os-nosdn-ovs-ha | | | X | | -+--------------------------+------+---------+------+------+ -| os-nosdn-ovs-noha | | | X | | -+--------------------------+------+---------+------+------+ -| os-ocl-nofeature-ha | | X | | | -+--------------------------+------+---------+------+------+ -| os-nosdn-lxd-ha | | | | X | -+--------------------------+------+---------+------+------+ -| os-nosdn-lxd-noha | | | | X | -+--------------------------+------+---------+------+------+ -| os-nosdn-fdio-ha | X | | | | -+--------------------------+------+---------+------+------+ -| os-odl_l2-fdio-noha | X | | | | -+--------------------------+------+---------+------+------+ -| os-odl-gluon-noha | X | | | | -+--------------------------+------+---------+------+------+ -| os-nosdn-openo-ha | | X | | | -+--------------------------+------+---------+------+------+ -| os-nosdn-kvm_ovs_dpdk | | | X | | -| -noha | | | | | -+--------------------------+------+---------+------+------+ -| os-nosdn-kvm_ovs_dpdk-ha | | | X | | -+--------------------------+------+---------+------+------+ -| os-nosdn-kvm_ovs_dpdk | | | X | | -| _bar-ha | | | | | -+--------------------------+------+---------+------+------+ -| os-nosdn-kvm_ovs_dpdk | | | X | | -| _bar-noha | | | | | -+--------------------------+------+---------+------+------+ -| opnfv_os-ovn-nofeature- | X | | | | -| noha_daily | | | | | -+--------------------------+------+---------+------+------+ +=============== + +For Fraser 6.0.0, Yardstick was tested on the following scenarios: + ++-------------------------+------+---------+----------+------+------+-------+ +| Scenario | Apex | Compass | Fuel-arm | Fuel | Joid | Daisy | ++=========================+======+=========+==========+======+======+=======+ +| os-nosdn-nofeature-noha | X | X | | | X | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-nosdn-nofeature-ha | X | X | X | X | X | X | ++-------------------------+------+---------+----------+------+------+-------+ +| os-nosdn-bar-noha | X | X | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-nosdn-bar-ha | X | | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-odl-bgpvpn-ha | X | | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-nosdn-calipso-noha | X | | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-nosdn-kvm-ha | | X | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-odl_l3-nofeature-ha | | X | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-odl-sfc-ha | | X | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| os-odl-nofeature-ha | | | | X | | X | ++-------------------------+------+---------+----------+------+------+-------+ +| os-nosdn-ovs-ha | | | | X | | | ++-------------------------+------+---------+----------+------+------+-------+ +| k8-nosdn-nofeature-ha | | X | | | | | ++-------------------------+------+---------+----------+------+------+-------+ +| k8-nosdn-stor4nfv-noha | | X | | | | | ++-------------------------+------+---------+----------+------+------+-------+ + Test results ------------- +============ Test results are available in: @@ -459,109 +365,107 @@ Test results are available in: The reporting pages can be found at: -+---------------+-------------------------------------------------------------------------------------+ -| apex | http://testresults.opnfv.org/reporting/euphrates/yardstick/status-apex.html | -+---------------+-------------------------------------------------------------------------------------+ -| compass | http://testresults.opnfv.org/reporting/euphrates/yardstick/status-compass.html | -+---------------+-------------------------------------------------------------------------------------+ -| fuel\@x86 | http://testresults.opnfv.org/reporting/euphrates/yardstick/status-fuel@x86.html | -+---------------+-------------------------------------------------------------------------------------+ -| fuel\@aarch64 | http://testresults.opnfv.org/reporting/euphrates/yardstick/status-fuel@aarch64.html | -+---------------+-------------------------------------------------------------------------------------+ -| joid | http://testresults.opnfv.org/reporting/euphrates/yardstick/status-joid.html | -+---------------+-------------------------------------------------------------------------------------+ ++---------------+----------------------------------------------------------------------------------+ +| apex | http://testresults.opnfv.org/reporting/fraser/yardstick/status-apex.html | ++---------------+----------------------------------------------------------------------------------+ +| compass | http://testresults.opnfv.org/reporting/fraser/yardstick/status-compass.html | ++---------------+----------------------------------------------------------------------------------+ +| fuel\@x86 | http://testresults.opnfv.org/reporting/fraser/yardstick/status-fuel@x86.html | ++---------------+----------------------------------------------------------------------------------+ +| fuel\@aarch64 | http://testresults.opnfv.org/reporting/fraser/yardstick/status-fuel@aarch64.html | ++---------------+----------------------------------------------------------------------------------+ +| joid | http://testresults.opnfv.org/reporting/fraser/yardstick/status-joid.html | ++---------------+----------------------------------------------------------------------------------+ Known Issues/Faults -^^^^^^^^^^^^^^^^^^^ +------------------- Corrected Faults -^^^^^^^^^^^^^^^^ +---------------- + +Fraser 6.0.0: + ++--------------------+--------------------------------------------------------------------------+ +| **JIRA REFERENCE** | **DESCRIPTION** | ++====================+==========================================================================+ +| YARDSTICK-831 | tc053 kill haproxy wrong | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-842 | load image fails when there's cirros image exist | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-857 | tc006 failed due to volume attached to different location "/dev/vdc" | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-874 | Specify supported architecture for Ubuntu backports repository | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-875 | Check if multiverse repository is available in Ubuntu | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-893 | Fix proxy env handling and ansible multinode support | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-899 | Variable local_iface_name is read before it is set | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-900 | Section in "upload_yardstick_image.yml" invalid | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-911 | Remove 'inconsistent-return-statements' from Pylint checks | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-989 | Yardstick real-time influxdb KPI reporting regressions | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-994 | NSB set-up build script for baremetal broken | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-996 | Error in address input format in "_ip_range_action_partial" | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1003 | Prox vnf descriptor cleanup for tg and vnf | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1006 | Ansible destroy script will fail if vm has already been undefined | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1012 | constants: fix pylint warnings for OSError | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1014 | Remove unused args in | +| | network_services.traffic_profile.ixia_rfc2544.IXIARFC2544Profile | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1016 | Allow vm to access outside world through default gateway | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1019 | For 'qemu-img version 2.10.1' unit 'MB' is not acceptable ansible script | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1021 | NSB: All Sample VNF test cases timeout after 1 hour of execution | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1036 | Prox: Addition of storage of extra counters for Grafana | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1038 | Missing file which is described in the operation_conf.yaml | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1047 | Error in string format in HeatTemplateError message | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1056 | yardstick report command print error when run test case | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1059 | Reduce the log level if TRex client is no connected | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1073 | Error when retrieving "options" section in "scenario" | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1080 | Running Test Case in Latest Yardstick Docker Image shows Error | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1082 | tc043,tc055, tc063, tc075, pass wrong node name in the ci scenario yaml | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1102 | Don't hide exception traceback from Task.start() | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1107 | bad exception traceback print due to atexit_handler | ++--------------------+--------------------------------------------------------------------------+ +| YARDSTICK-1120 | HA test case tc050 should start monitor before attack | ++--------------------+--------------------------------------------------------------------------+ + +Fraser 6.0.0 known restrictions/issues +====================================== -Euphrates 5.1.0: - -+---------------------+-------------------------------------------------------------------------+ -| **JIRA REFERENCE** | **DESCRIPTION** | -| | | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-841 | Fix various NSB license issues | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-73 | How To Work with Test Cases | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-500 | VNF testing documentation | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-826 | Allow overriding Heat IP addresses to match traffic generator profile | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-828 | Refactor doc/testing/user/userguide "Yardstick Installation" | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-830 | build_yardstick_image Ansible mount module doesn't work on Ubuntu 14.04 | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-833 | ansible_common transform password into lower case | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-847 | tc006, tc079, tc082 miss grafana dashboard in local deployment | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-849 | kill process do not accurately kill the process like "nova-api" | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-850 | tc023 miss description and tc050-58 wrong description | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-852 | tc078 cpu2006 fails in some situation | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-854 | yardstick docker lack of trex_client | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-867 | testcase tc078 have no data stored or dashboard to show results | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-871 | Remove img_modify_playbook assignation in build_yardstick_image.yml | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-829 | "nsb_setup.sh" doesn't parse the controller IP correctly | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-839 | NSB Prox BM test cases to be fixed for incorporating scale-up | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-840 | NSB Prox test documentation of vPE and LW-AFTR test cases | -+---------------------+-------------------------------------------------------------------------+ -| JIRA: YARDSTICK-848 | NSB "Prox" : Cleanup duplicated traffic profile | -+---------------------+-------------------------------------------------------------------------+ - - - - -Euphrates 5.0.0: - -+---------------------+--------------------------------------------+ -| **JIRA REFERENCE** | **DESCRIPTION** | -| | | -+---------------------+--------------------------------------------+ -| JIRA: YARDSTICK-599 | Could not load EntryPoint.parse when using | -| | 'openstack -h' | -+---------------------+--------------------------------------------+ -| JIRA: YARDSTICK-602 | Don't rely on staic ip addresses as they | -| | are dynamic | -+---------------------+--------------------------------------------+ - - -Euphratess 5.0.0 known restrictions/issues ------------------------------------------- +-----------+-----------+----------------------------------------------+ | Installer | Scenario | Issue | +===========+===========+==============================================+ -| any | \*-bgpvpn | Floating ips not supported. Some Test cases | -| | | related to floating ips are excluded. | -+-----------+-----------+----------------------------------------------+ -| any | odl_l3-\* | Some test cases related to using floating IP | -| | | addresses fail because of a known ODL bug. | -| | | | -+-----------+-----------+----------------------------------------------+ -| compass | odl_l2-\* | In some test cases, VM instance will failed | -| | | raising network interfaces. | | | | | +-----------+-----------+----------------------------------------------+ - Useful links ------------- +============ - wiki project page: https://wiki.opnfv.org/display/yardstick/Yardstick - - wiki Yardstick Euphrates release planing page: https://wiki.opnfv.org/display/yardstick/Yardstick+Euphrates+Release+Planning + - wiki Yardstick Fraser release planing page: https://wiki.opnfv.org/display/yardstick/Release+Fraser - Yardstick repo: https://git.opnfv.org/cgit/yardstick diff --git a/docs/testing/user/userguide/01-introduction.rst b/docs/testing/user/userguide/01-introduction.rst index c1d5def98..d846e759c 100755 --- a/docs/testing/user/userguide/01-introduction.rst +++ b/docs/testing/user/userguide/01-introduction.rst @@ -42,43 +42,47 @@ This document consists of the following chapters: * Chapter :doc:`02-methodology` describes the methodology implemented by the *Yardstick* Project for :term:`NFVI` verification. -* Chapter :doc:`03-architecture` provides information on the software architecture - of *Yardstick*. +* Chapter :doc:`03-architecture` provides information on the software + architecture of *Yardstick*. * Chapter :doc:`04-installation` provides instructions to install *Yardstick*. -* Chapter :doc:`05-yardstick_plugin` provides information on how to integrate +* Chapter :doc:`05-operation` provides information on how to use *Yardstick* + to run and create testcases. + +* Chapter :doc:`06-yardstick-plugin` provides information on how to integrate other OPNFV testing projects into *Yardstick*. -* Chapter :doc:`06-result-store-InfluxDB` provides inforamtion on how to run +* Chapter :doc:`07-result-store-InfluxDB` provides inforamtion on how to run plug-in test cases and store test results into community's InfluxDB. -* Chapter :doc:`07-grafana` provides inforamtion on *Yardstick* grafana dashboard - and how to add a dashboard into *Yardstick* grafana dashboard. +* Chapter :doc:`08-grafana` provides inforamtion on *Yardstick* grafana + dashboard and how to add a dashboard into *Yardstick* grafana dashboard. -* Chapter :doc:`08-api` provides inforamtion on *Yardstick* ReST API and how to +* Chapter :doc:`09-api` provides inforamtion on *Yardstick* ReST API and how to use *Yardstick* API. -* Chapter :doc:`09-yardstick_user_interface` provides inforamtion on how to use +* Chapter :doc:`10-yardstick-user-interface` provides inforamtion on how to use yardstick report CLI to view the test result in table format and also values pinned on to a graph -* Chapter :doc:`10-vtc-overview` provides information on the :term:`VTC`. +* Chapter :doc:`11-vtc-overview` provides information on the :term:`VTC`. -* Chapter :doc:`13-nsb-overview` describes the methodology implemented by the +* Chapter :doc:`12-nsb-overview` describes the methodology implemented by the Yardstick - Network service benchmarking to test real world usecase for a given VNF. -* Chapter :doc:`14-nsb_installation` provides instructions to install - *Yardstick - Network service benchmarking testing*. +* Chapter :doc:`13-nsb_installation` provides instructions to install + *Yardstick - Network Service Benchmarking (NSB) testing*. + +* Chapter :doc:`14-nsb-operation` provides information on running *NSB* * Chapter :doc:`15-list-of-tcs` includes a list of available *Yardstick* test cases. - Contact Yardstick ================= Feedback? `Contact us`_ -.. _Contact us: opnfv-users@lists.opnfv.org +.. _Contact us: mailto:opnfv-users@lists.opnfv.org&subject="[yardstick]" diff --git a/docs/testing/user/userguide/03-architecture.rst b/docs/testing/user/userguide/03-architecture.rst index 8336b609d..622002ee4 100755 --- a/docs/testing/user/userguide/03-architecture.rst +++ b/docs/testing/user/userguide/03-architecture.rst @@ -9,8 +9,9 @@ Architecture Abstract ======== -This chapter describes the yardstick framework software architecture. we will introduce it from Use-Case View, -Logical View, Process View and Deployment View. More technical details will be introduced in this chapter. +This chapter describes the yardstick framework software architecture. We will +introduce it from Use-Case View, Logical View, Process View and Deployment +View. More technical details will be introduced in this chapter. Overview ======== @@ -23,8 +24,8 @@ files. Yardstick is inspired by Rally. Yardstick is intended to run on a computer with access and credentials to a cloud. The test case is described in a configuration file given as an argument. -How it works: the benchmark task configuration file is parsed and converted into -an internal model. The context part of the model is converted into a Heat +How it works: the benchmark task configuration file is parsed and converted +into an internal model. The context part of the model is converted into a Heat template and deployed into a stack. Each scenario is run using a runner, either serially or in parallel. Each runner runs in its own subprocess executing commands in a VM using SSH. The output of each scenario is written as json @@ -43,13 +44,15 @@ names, image names, affinity rules and network configurations. A context is converted into a simplified Heat template, which is used to deploy onto the Openstack environment. -**Data** - Output produced by running a benchmark, written to a file in json format +**Data** - Output produced by running a benchmark, written to a file in json +format **Runner** - Logic that determines how a test scenario is run and reported, for example the number of test iterations, input value stepping and test duration. Predefined runner types exist for re-usage, see `Runner types`_. -**Scenario** - Type/class of measurement for example Ping, Pktgen, (Iperf, LmBench, ...) +**Scenario** - Type/class of measurement for example Ping, Pktgen, (Iperf, +LmBench, ...) **SLA** - Relates to what result boundary a test case must meet to pass. For example a latency limit, amount or ratio of lost packets and so on. Action @@ -128,8 +131,8 @@ Snippet of an Iteration runner configuration: Use-Case View ============= Yardstick Use-Case View shows two kinds of users. One is the Tester who will -do testing in cloud, the other is the User who is more concerned with test result -and result analyses. +do testing in cloud, the other is the User who is more concerned with test +result and result analyses. For testers, they will run a single test case or test case suite to verify infrastructure compliance or bencnmark their own infrastructure performance. @@ -254,7 +257,8 @@ Yardstick Directory structure *tools/* - Currently contains tools to build image for VMs which are deployed by Heat. Currently contains how to build the yardstick-trusty-server - image with the different tools that are needed from within the image. + image with the different tools that are needed from within the + image. *plugin/* - Plug-in configuration files are stored here. diff --git a/docs/testing/user/userguide/04-installation.rst b/docs/testing/user/userguide/04-installation.rst index cac814667..a4846230e 100644 --- a/docs/testing/user/userguide/04-installation.rst +++ b/docs/testing/user/userguide/04-installation.rst @@ -39,18 +39,18 @@ Several prerequisites are needed for Yardstick: 4. Connectivity from the Jumphost to the SUT public/external network .. note:: *Jumphost* refers to any server which meets the previous -requirements. Normally it is the same server from where the OPNFV -deployment has been triggered. + requirements. Normally it is the same server from where the OPNFV + deployment has been triggered. .. warning:: Connectivity from Jumphost is essential and it is of paramount -importance to make sure it is working before even considering to install -and run Yardstick. Make also sure you understand how your networking is -designed to work. + importance to make sure it is working before even considering to install + and run Yardstick. Make also sure you understand how your networking is + designed to work. .. note:: If your Jumphost is operating behind a company http proxy and/or -Firewall, please first consult `Proxy Support`_ section which is towards the -end of this document. That section details some tips/tricks which *may* be of -help in a proxified environment. + Firewall, please first consult `Proxy Support`_ section which is towards + the end of this document. That section details some tips/tricks which *may* + be of help in a proxified environment. Install Yardstick using Docker (first option) (**recommended**) @@ -85,27 +85,30 @@ Run the Docker image to get a Yardstick container:: docker run -itd --privileged -v /var/run/docker.sock:/var/run/docker.sock \ -p 8888:5000 --name yardstick opnfv/yardstick:stable -.. table:: Description of the parameters used with ``docker run`` command - - ======================= ==================================================== - Parameters Detail - ======================= ==================================================== - -itd -i: interactive, Keep STDIN open even if not - attached - -t: allocate a pseudo-TTY detached mode, in the - background - ======================= ==================================================== - --privileged If you want to build ``yardstick-image`` in - Yardstick container, this parameter is needed - ======================= ==================================================== - -p 8888:5000 Redirect the a host port (8888) to a container port - (5000) - ======================= ==================================================== - -v /var/run/docker.sock If you want to use yardstick env grafana/influxdb to - :/var/run/docker.sock create a grafana/influxdb container out of Yardstick - container - ======================= ==================================================== - --name yardstick The name for this container +Description of the parameters used with ``docker run`` command + + +------------------------+--------------------------------------------------+ + | Parameters | Detail | + +========================+==================================================+ + | -itd | -i: interactive, Keep STDIN open even if not | + | | attached | + | +--------------------------------------------------+ + | | -t: allocate a pseudo-TTY detached mode, in the | + | | background | + +------------------------+--------------------------------------------------+ + | --privileged | If you want to build ``yardstick-image`` in | + | | Yardstick container, this parameter is needed | + +------------------------+--------------------------------------------------+ + | -p 8888:5000 | Redirect the a host port (8888) to a container | + | | port (5000) | + +------------------------+--------------------------------------------------+ + | -v /var/run/docker.sock| If you want to use yardstick env | + | :/var/run/docker.sock | grafana/influxdb to create a grafana/influxdb | + | | container out of Yardstick container | + +------------------------+--------------------------------------------------+ + | --name yardstick | The name for this container | + +------------------------+--------------------------------------------------+ + If the host is restarted ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -135,18 +138,18 @@ automatically:: yardstick env prepare .. note:: Since Euphrates release, the above command will not be able to -automatically configure the ``/etc/yardstick/openstack.creds`` file. So before -running the above command, it is necessary to create the -``/etc/yardstick/openstack.creds`` file and save OpenStack environment -variables into it manually. If you have the openstack credential file saved -outside the Yardstick Docker container, you can do this easily by mapping the -credential file into Yardstick container using:: + automatically configure the ``/etc/yardstick/openstack.creds`` file. So before + running the above command, it is necessary to create the + ``/etc/yardstick/openstack.creds`` file and save OpenStack environment + variables into it manually. If you have the openstack credential file saved + outside the Yardstick Docker container, you can do this easily by mapping the + credential file into Yardstick container using:: - '-v /path/to/credential_file:/etc/yardstick/openstack.creds' + '-v /path/to/credential_file:/etc/yardstick/openstack.creds' -when running the Yardstick container. For details of the required OpenStack -environment variables please refer to section `Export OpenStack environment -variables`_. + when running the Yardstick container. For details of the required OpenStack + environment variables please refer to section `Export OpenStack environment + variables`_. The ``env prepare`` command may take up to 6-8 minutes to finish building yardstick-image and other environment preparation. Meanwhile if you wish to @@ -222,8 +225,8 @@ Yardstick is installed:: sudo -EH tools/yardstick-img-modify tools/ubuntu-server-cloudimg-modify.sh .. warning:: Before building the guest image inside the Yardstick container, -make sure the container is granted with privilege. The script will create files -by default in ``/tmp/workspace/yardstick`` and the files will be owned by root. + make sure the container is granted with privilege. The script will create files + by default in ``/tmp/workspace/yardstick`` and the files will be owned by root. The created image can be added to OpenStack using the OpenStack client or via the OpenStack Dashboard:: @@ -270,7 +273,7 @@ For usage of Yardstick GUI, please watch our demo video at `Yardstick GUI demo`_. .. note:: The Yardstick GUI is still in development, the GUI layout and -features may change. + features may change. Delete the Yardstick container ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -433,7 +436,7 @@ of Yardstick ``help`` command and ``ping.py`` test sample:: yardstick task start samples/ping.yaml .. note:: The above commands could be run in both the Yardstick container and -the Ubuntu directly. + the Ubuntu directly. Each testing tool supported by Yardstick has a sample configuration file. These configuration files can be found in the ``samples`` directory. @@ -468,10 +471,10 @@ Then you can run a test case and visit http://host_ip:1948 (``admin``/``admin``) to see the results. .. note:: Executing ``yardstick env`` command to deploy InfluxDB and Grafana -requires Jumphost's docker API version => 1.24. Run the following command to -check the docker API version on the Jumphost:: + requires Jumphost's docker API version => 1.24. Run the following command to + check the docker API version on the Jumphost:: - docker version + docker version Manual deployment of InfluxDB and Grafana containers @@ -537,200 +540,6 @@ Deploy InfluxDB and Grafana directly in Ubuntu (**Todo**) --------------------------------------------------------- -Yardstick common CLI --------------------- - -List test cases -^^^^^^^^^^^^^^^ - -``yardstick testcase list``: This command line would list all test cases in -Yardstick. It would show like below:: - - +--------------------------------------------------------------------------------------- - | Testcase Name | Description - +--------------------------------------------------------------------------------------- - | opnfv_yardstick_tc001 | Measure network throughput using pktgen - | opnfv_yardstick_tc002 | measure network latency using ping - | opnfv_yardstick_tc005 | Measure Storage IOPS, throughput and latency using fio. - ... - +--------------------------------------------------------------------------------------- - - -Show a test case config file -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Take opnfv_yardstick_tc002 for an example. This test case measure network -latency. You just need to type in ``yardstick testcase show -opnfv_yardstick_tc002``, and the console would show the config yaml of this -test case:: - - --- - - schema: "yardstick:task:0.1" - description: > - Yardstick TC002 config file; - measure network latency using ping; - - {% set image = image or "cirros-0.3.5" %} - - {% set provider = provider or none %} - {% set physical_network = physical_network or 'physnet1' %} - {% set segmentation_id = segmentation_id or none %} - {% set packetsize = packetsize or 100 %} - - scenarios: - {% for i in range(2) %} - - - type: Ping - options: - packetsize: {{packetsize}} - host: athena.demo - target: ares.demo - - runner: - type: Duration - duration: 60 - interval: 10 - - sla: - max_rtt: 10 - action: monitor - {% endfor %} - - context: - name: demo - image: {{image}} - flavor: yardstick-flavor - user: cirros - - placement_groups: - pgrp1: - policy: "availability" - - servers: - athena: - floating_ip: true - placement: "pgrp1" - ares: - placement: "pgrp1" - - networks: - test: - cidr: '10.0.1.0/24' - {% if provider == "vlan" %} - provider: {{provider}} - physical_network: {{physical_network}}å - {% if segmentation_id %} - segmentation_id: {{segmentation_id}} - {% endif %} - {% endif %} - - -Start a task to run yardstick test case -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want run a test case, then you need to use ``yardstick task start -<test_case_path>`` this command support some parameters as below:: - - +---------------------+--------------------------------------------------+ - | Parameters | Detail | - +=====================+==================================================+ - | -d | show debug log of yardstick running | - | | | - +---------------------+--------------------------------------------------+ - | --task-args | If you want to customize test case parameters, | - | | use "--task-args" to pass the value. The format | - | | is a json string with parameter key-value pair. | - | | | - +---------------------+--------------------------------------------------+ - | --task-args-file | If you want to use yardstick | - | | env prepare command(or | - | | related API) to load the | - +---------------------+--------------------------------------------------+ - | --parse-only | | - | | | - | | | - +---------------------+--------------------------------------------------+ - | --output-file \ | Specify where to output the log. if not pass, | - | OUTPUT_FILE_PATH | the default value is | - | | "/tmp/yardstick/yardstick.log" | - | | | - +---------------------+--------------------------------------------------+ - | --suite \ | run a test suite, TEST_SUITE_PATH specify where | - | TEST_SUITE_PATH | the test suite locates | - | | | - +---------------------+--------------------------------------------------+ - - -Run Yardstick in a local environment ------------------------------------- - -We also have a guide about how to run Yardstick in a local environment. -This work is contributed by Tapio Tallgren. -You can find this guide at `How to run Yardstick in a local environment`_. - - -Create a test suite for Yardstick ------------------------------------- - -A test suite in yardstick is a yaml file which include one or more test cases. -Yardstick is able to support running test suite task, so you can customize your -own test suite and run it in one task. - -``tests/opnfv/test_suites`` is the folder where Yardstick puts CI test suite. -A typical test suite is like below (the ``fuel_test_suite.yaml`` example):: - - --- - # Fuel integration test task suite - - schema: "yardstick:suite:0.1" - - name: "fuel_test_suite" - test_cases_dir: "samples/" - test_cases: - - - file_name: ping.yaml - - - file_name: iperf3.yaml - -As you can see, there are two test cases in the ``fuel_test_suite.yaml``. The -``schema`` and the ``name`` must be specified. The test cases should be listed -via the tag ``test_cases`` and their relative path is also marked via the tag -``test_cases_dir``. - -Yardstick test suite also supports constraints and task args for each test -case. Here is another sample (the ``os-nosdn-nofeature-ha.yaml`` example) to -show this, which is digested from one big test suite:: - - --- - - schema: "yardstick:suite:0.1" - - name: "os-nosdn-nofeature-ha" - test_cases_dir: "tests/opnfv/test_cases/" - test_cases: - - - file_name: opnfv_yardstick_tc002.yaml - - - file_name: opnfv_yardstick_tc005.yaml - - - file_name: opnfv_yardstick_tc043.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/.../pod.yaml", - "host": "node4.LF","target": "node5.LF"}' - -As you can see in test case ``opnfv_yardstick_tc043.yaml``, there are two -tags, ``constraint`` and ``task_args``. ``constraint`` is to specify which -installer or pod it can be run in the CI environment. ``task_args`` is to -specify the task arguments for each pod. - -All in all, to create a test suite in Yardstick, you just need to create a -yaml file and add test cases, constraint or task arguments if necessary. - - Proxy Support ------------- @@ -790,7 +599,7 @@ stop and delete the container:: sudo docker rm yardstick .. warning:: Be careful, the above ``rm`` command will delete the container -completely. Everything on this container will be lost. + completely. Everything on this container will be lost. Then follow the previous instructions `Prepare the Yardstick container`_ to rebuild the Yardstick container. @@ -804,4 +613,3 @@ References .. _`Cirros 0.3.5`: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img .. _`Ubuntu 16.04`: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img .. _`Yardstick GUI demo`: https://www.youtube.com/watch?v=M3qbJDp6QBk -.. _`How to run Yardstick in a local environment`: https://wiki.opnfv.org/display/yardstick/How+to+run+Yardstick+in+a+local+environment diff --git a/docs/testing/user/userguide/05-operation.rst b/docs/testing/user/userguide/05-operation.rst new file mode 100644 index 000000000..f390d1643 --- /dev/null +++ b/docs/testing/user/userguide/05-operation.rst @@ -0,0 +1,296 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Intel, Ericsson AB, Huawei Technologies Co. Ltd and others. + +.. + Convention for heading levels in Yardstick: + ======= Heading 0 (reserved for the title in a document) + ------- Heading 1 + ^^^^^^^ Heading 2 + +++++++ Heading 3 + ''''''' Heading 4 + Avoid deeper levels because they do not render well. + +=============== +Yardstick Usage +=============== + +Once you have yardstick installed, you can start using it to run testcases +immediately, through the CLI. You can also define and run new testcases and +test suites. This chapter details basic usage (running testcases), as well as +more advanced usage (creating your own testcases). + +Yardstick common CLI +-------------------- + +List test cases +^^^^^^^^^^^^^^^ + +``yardstick testcase list``: This command line would list all test cases in +Yardstick. It would show like below:: + + +--------------------------------------------------------------------------------------- + | Testcase Name | Description + +--------------------------------------------------------------------------------------- + | opnfv_yardstick_tc001 | Measure network throughput using pktgen + | opnfv_yardstick_tc002 | measure network latency using ping + | opnfv_yardstick_tc005 | Measure Storage IOPS, throughput and latency using fio. + ... + +--------------------------------------------------------------------------------------- + + +Show a test case config file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Take opnfv_yardstick_tc002 for an example. This test case measure network +latency. You just need to type in ``yardstick testcase show +opnfv_yardstick_tc002``, and the console would show the config yaml of this +test case: + +.. literalinclude:: + ../../../../tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml + :lines: 9- + +Run a Yardstick test case +^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want run a test case, then you need to use ``yardstick task start +<test_case_path>`` this command support some parameters as below: + + +---------------------+--------------------------------------------------+ + | Parameters | Detail | + +=====================+==================================================+ + | -d | show debug log of yardstick running | + | | | + +---------------------+--------------------------------------------------+ + | --task-args | If you want to customize test case parameters, | + | | use "--task-args" to pass the value. The format | + | | is a json string with parameter key-value pair. | + | | | + +---------------------+--------------------------------------------------+ + | --task-args-file | If you want to use yardstick | + | | env prepare command(or | + | | related API) to load the | + +---------------------+--------------------------------------------------+ + | --parse-only | | + | | | + | | | + +---------------------+--------------------------------------------------+ + | --output-file \ | Specify where to output the log. if not pass, | + | OUTPUT_FILE_PATH | the default value is | + | | "/tmp/yardstick/yardstick.log" | + | | | + +---------------------+--------------------------------------------------+ + | --suite \ | run a test suite, TEST_SUITE_PATH specify where | + | TEST_SUITE_PATH | the test suite locates | + | | | + +---------------------+--------------------------------------------------+ + + +Run Yardstick in a local environment +------------------------------------ + +We also have a guide about `How to run Yardstick in a local environment`_. +This work is contributed by Tapio Tallgren. + +Create a new testcase for Yardstick +----------------------------------- + +As a user, you may want to define a new testcase in addition to the ones +already available in Yardstick. This section will show you how to do this. + +Each testcase consists of two sections: + +* ``scenarios`` describes what will be done by the test +* ``context`` describes the environment in which the test will be run. + +Defining the testcase scenarios +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +TODO + +Defining the testcase context(s) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each testcase consists of one or more contexts, which describe the environment +in which the testcase will be run. +Current available contexts are: + +* ``Dummy``: this is a no-op context, and is used when there is no environment + to set up e.g. when testing whether OpenStack services are available +* ``Node``: this context is used to perform operations on baremetal servers +* ``Heat``: uses OpenStack to provision the required hosts, networks, etc. +* ``Kubernetes``: uses Kubernetes to provision the resources required for the + test. + +Regardless of the context type, the ``context`` section of the testcase will +consist of the following:: + + context: + name: demo + type: Dummy|Node|Heat|Kubernetes + +The content of the ``context`` section will vary based on the context type. + +Dummy Context ++++++++++++++ + +No additional information is required for the Dummy context:: + + context: + name: my_context + type: Dummy + +Node Context +++++++++++++ + +TODO + +Heat Context +++++++++++++ + +In addition to ``name`` and ``type``, a Heat context requires the following +arguments: + +* ``image``: the image to be used to boot VMs +* ``flavor``: the flavor to be used for VMs in the context +* ``user``: the username for connecting into the VMs +* ``networks``: The networks to be created, networks are identified by name + + * ``name``: network name (required) + * (TODO) Any optional attributes + +* ``servers``: The servers to be created + + * ``name``: server name + * (TODO) Any optional attributes + +In addition to the required arguments, the following optional arguments can be +passed to the Heat context: + +* ``placement_groups``: + + * ``name``: the name of the placement group to be created + * ``policy``: either ``affinity`` or ``availability`` +* ``server_groups``: + + * ``name``: the name of the server group + * ``policy``: either ``affinity`` or ``anti-affinity`` + +Combining these elements together, a sample Heat context config looks like: + +.. literalinclude:: + ../../../../yardstick/tests/integration/dummy-scenario-heat-context.yaml + :start-after: --- + :empahsise-lines: 14- + +Using exisiting HOT Templates +''''''''''''''''''''''''''''' + +TODO + +Kubernetes Context +++++++++++++++++++ + +TODO + +Using multiple contexts in a testcase ++++++++++++++++++++++++++++++++++++++ + +When using multiple contexts in a testcase, the ``context`` section is replaced +by a ``contexts`` section, and each context is separated with a ``-`` line:: + + contexts: + - + name: context1 + type: Heat + ... + - + name: context2 + type: Node + ... + + +Reusing a context ++++++++++++++++++ + +Typically, a context is torn down after a testcase is run, however, the user +may wish to keep an context intact after a testcase is complete. + +.. note:: + This feature has been implemented for the Heat context only + +To keep or reuse a context, the ``flags`` option must be specified: + +* ``no_setup``: skip the deploy stage, and fetch the details of a deployed + context/Heat stack. +* ``no_teardown``: skip the undeploy stage, thus keeping the stack intact for + the next test + +If either of these ``flags`` are ``True``, the context information must still +be given. By default, these flags are disabled:: + + context: + name: mycontext + type: Heat + flags: + no_setup: True + no_teardown: True + ... + +Create a test suite for Yardstick +--------------------------------- + +A test suite in Yardstick is a .yaml file which includes one or more test +cases. Yardstick is able to support running test suite task, so you can +customize your own test suite and run it in one task. + +``tests/opnfv/test_suites`` is the folder where Yardstick puts CI test suite. +A typical test suite is like below (the ``fuel_test_suite.yaml`` example): + +.. literalinclude:: + ../../../../tests/opnfv/test_suites/fuel_test_suite.yaml + :lines: 9- + +As you can see, there are two test cases in the ``fuel_test_suite.yaml``. The +``schema`` and the ``name`` must be specified. The test cases should be listed +via the tag ``test_cases`` and their relative path is also marked via the tag +``test_cases_dir``. + +Yardstick test suite also supports constraints and task args for each test +case. Here is another sample (the ``os-nosdn-nofeature-ha.yaml`` example) to +show this, which is digested from one big test suite:: + + --- + + schema: "yardstick:suite:0.1" + + name: "os-nosdn-nofeature-ha" + test_cases_dir: "tests/opnfv/test_cases/" + test_cases: + - + file_name: opnfv_yardstick_tc002.yaml + - + file_name: opnfv_yardstick_tc005.yaml + - + file_name: opnfv_yardstick_tc043.yaml + constraint: + installer: compass + pod: huawei-pod1 + task_args: + huawei-pod1: '{"pod_info": "etc/yardstick/.../pod.yaml", + "host": "node4.LF","target": "node5.LF"}' + +As you can see in test case ``opnfv_yardstick_tc043.yaml``, there are two +tags, ``constraint`` and ``task_args``. ``constraint`` is to specify which +installer or pod it can be run in the CI environment. ``task_args`` is to +specify the task arguments for each pod. + +All in all, to create a test suite in Yardstick, you just need to create a +yaml file and add test cases, constraint or task arguments if necessary. + +References +---------- + +.. _`How to run Yardstick in a local environment`: https://wiki.opnfv.org/display/yardstick/How+to+run+Yardstick+in+a+local+environment diff --git a/docs/testing/user/userguide/05-yardstick_plugin.rst b/docs/testing/user/userguide/06-yardstick-plugin.rst index 679ce7900..bc35e239d 100644 --- a/docs/testing/user/userguide/05-yardstick_plugin.rst +++ b/docs/testing/user/userguide/06-yardstick-plugin.rst @@ -31,7 +31,7 @@ In this introduction we will install Storperf on Jump Host. Step 0: Environment preparation ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +------------------------------- Running Storperf on Jump Host Requirements: @@ -47,24 +47,26 @@ environment and other dependencies: 1. Make sure docker is installed. 2. Make sure Keystone, Nova, Neutron, Glance, Heat are installed correctly. 3. Make sure Jump Host have access to the OpenStack Controller API. -4. Make sure Jump Host must have internet connectivity for downloading docker image. -5. You need to know where to get basic openstack Keystone authorization info, such as - OS_PASSWORD, OS_PROJECT_NAME, OS_AUTH_URL, OS_USERNAME. -6. To run a Storperf container, you need to have OpenStack Controller environment - variables defined and passed to Storperf container. The best way to do this is to - put environment variables in a "storperf_admin-rc" file. The storperf_admin-rc - should include credential environment variables at least: - -* OS_AUTH_URL -* OS_USERNAME -* OS_PASSWORD -* OS_PROJECT_NAME -* OS_PROJECT_ID -* OS_USER_DOMAIN_ID - -*Yardstick* has a "prepare_storperf_admin-rc.sh" script which can be used to -generate the "storperf_admin-rc" file, this script is located at -test/ci/prepare_storperf_admin-rc.sh +4. Make sure Jump Host must have internet connectivity for downloading docker + image. +5. You need to know where to get basic openstack Keystone authorization info, + such as OS_PASSWORD, OS_PROJECT_NAME, OS_AUTH_URL, OS_USERNAME. +6. To run a Storperf container, you need to have OpenStack Controller + environment variables defined and passed to Storperf container. The best way + to do this is to put environment variables in a "storperf_admin-rc" file. + The storperf_admin-rc should include credential environment variables at + least: + + * OS_AUTH_URL + * OS_USERNAME + * OS_PASSWORD + * OS_PROJECT_NAME + * OS_PROJECT_ID + * OS_USER_DOMAIN_ID + +*Yardstick* has a ``prepare_storperf_admin-rc.sh`` script which can be used to +generate the ``storperf_admin-rc`` file, this script is located at +``test/ci/prepare_storperf_admin-rc.sh`` :: @@ -92,18 +94,18 @@ test/ci/prepare_storperf_admin-rc.sh echo "OS_USER_DOMAIN_ID="$USER_DOMAIN_ID >> ~/storperf_admin-rc -The generated "storperf_admin-rc" file will be stored in the root directory. If -you installed *Yardstick* using Docker, this file will be located in the +The generated ``storperf_admin-rc`` file will be stored in the root directory. +If you installed *Yardstick* using Docker, this file will be located in the container. You may need to copy it to the root directory of the Storperf deployed host. Step 1: Plug-in configuration file preparation ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +---------------------------------------------- To install a plug-in, first you need to prepare a plug-in configuration file in -YAML format and store it in the "plugin" directory. The plugin configration file -work as the input of yardstick "plugin" command. Below is the Storperf plug-in -configuration file sample: +YAML format and store it in the "plugin" directory. The plugin configration +file work as the input of yardstick "plugin" command. Below is the Storperf +plug-in configuration file sample: :: --- @@ -123,28 +125,28 @@ Here the Storperf will be installed on IP 192.168.23.2 which is the Jump Host in my local environment. Step 2: Plug-in install/remove scripts preparation ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +-------------------------------------------------- -In "yardstick/resource/scripts" directory, there are two folders: a "install" -folder and a "remove" folder. You need to store the plug-in install/remove -scripts in these two folders respectively. +In ``yardstick/resource/scripts`` directory, there are two folders: an +``install`` folder and a ``remove`` folder. You need to store the plug-in +install/remove scripts in these two folders respectively. The detailed installation or remove operation should de defined in these two scripts. The name of both install and remove scripts should match the plugin-in name that you specified in the plug-in configuration file. -For example, the install and remove scripts for Storperf are both named to -"storperf.bash". +For example, the install and remove scripts for Storperf are both named +``storperf.bash``. Step 3: Install and remove Storperf ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +----------------------------------- To install Storperf, simply execute the following command:: # Install Storperf yardstick plugin install plugin/storperf.yaml -removing Storperf from yardstick +Removing Storperf from yardstick ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To remove Storperf, simply execute the following command:: diff --git a/docs/testing/user/userguide/06-result-store-InfluxDB.rst b/docs/testing/user/userguide/07-result-store-InfluxDB.rst index 747927889..cde931376 100644 --- a/docs/testing/user/userguide/06-result-store-InfluxDB.rst +++ b/docs/testing/user/userguide/07-result-store-InfluxDB.rst @@ -24,7 +24,7 @@ Store Storperf Test Results into Community's InfluxDB ===================================================== .. _Influxdb: https://git.opnfv.org/cgit/yardstick/tree/yardstick/dispatcher/influxdb.py -.. _Mingjiang: limingjiang@huawei.com +.. _Mingjiang: mailto:limingjiang@huawei.com .. _Visual: https://wiki.opnfv.org/download/attachments/6827660/tc074.PNG?version=1&modificationDate=1470298075000&api=v2 .. _Login: http://testresults.opnfv.org/grafana/login @@ -40,12 +40,13 @@ into community's InfluxDB: will be supported in the future. Our plan is to support rest-api in D release so that other testing projects can -call the rest-api to use yardstick dispatcher service to push data to yardstick's -influxdb database. +call the rest-api to use yardstick dispatcher service to push data to +Yardstick's InfluxDB database. -For now, influxdb only support line protocol, and the json protocol is deprecated. +For now, InfluxDB only supports line protocol, and the json protocol is +deprecated. -Take ping test case for example, the raw_result is json format like this: +Take ping test case for example, the ``raw_result`` is json format like this: :: "benchmark": { @@ -61,23 +62,24 @@ Take ping test case for example, the raw_result is json format like this: "runner_id": 2625 } -With the help of "influxdb_line_protocol", the json is transform to like below as a line string: -:: +With the help of "influxdb_line_protocol", the json is transform to like below +as a line string:: 'ping,deploy_scenario=unknown,host=athena.demo,installer=unknown,pod_name=unknown, runner_id=2625,scenarios=Ping,target=ares.demo,task_id=77755f38-1f6a-4667-a7f3- 301c99963656,version=unknown rtt.ares=1.125 1470315409868094976' -So, for data output of json format, you just need to transform json into line format and call -influxdb api to post the data into the database. All this function has been implemented in Influxdb_. -If you need support on this, please contact Mingjiang_. +So, for data output of json format, you just need to transform json into line +format and call influxdb api to post the data into the database. All this +function has been implemented in Influxdb_. If you need support on this, please +contact Mingjiang_. :: curl -i -XPOST 'http://104.197.68.199:8086/write?db=yardstick' -- data-binary 'ping,deploy_scenario=unknown,host=athena.demo,installer=unknown, ...' -Grafana will be used for visualizing the collected test data, which is shown in Visual_. Grafana -can be accessed by Login_. +Grafana will be used for visualizing the collected test data, which is shown in +Visual_. Grafana can be accessed by Login_. .. image:: images/results_visualization.png diff --git a/docs/testing/user/userguide/07-grafana.rst b/docs/testing/user/userguide/08-grafana.rst index 416857b71..29bc23a08 100644 --- a/docs/testing/user/userguide/07-grafana.rst +++ b/docs/testing/user/userguide/08-grafana.rst @@ -108,8 +108,10 @@ There are 6 steps to go. 5. When finished with all Grafana configuration changes in this temporary dashboard then chose "export" of the updated dashboard copy into a JSON file - and put it up for review in Gerrit, in file /yardstick/dashboard/Yardstick-TCxxx-yyyyyyyyyyyyy. - For instance a typical default name of the file would be "Yardstick-TC001 Copy-1234567891234". + and put it up for review in Gerrit, in file + ``/yardstick/dashboard/Yardstick-TCxxx-yyyyyyyyyyyyy``. + For instance a typical default name of the file would be + ``Yardstick-TC001 Copy-1234567891234``. 6. Once you finish your dashboard, the next step is exporting the configuration file and propose a patch into Yardstick. Yardstick team will review and diff --git a/docs/testing/user/userguide/08-api.rst b/docs/testing/user/userguide/09-api.rst index 2206c2ac8..f0ae3980b 100644 --- a/docs/testing/user/userguide/08-api.rst +++ b/docs/testing/user/userguide/09-api.rst @@ -3,25 +3,29 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Huawei Technologies Co.,Ltd and others. +===================== Yardstick Restful API -====================== +===================== Abstract --------- +======== Yardstick support restful API since Danube. Available API -------------- +============= /yardstick/env/action -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------- -Description: This API is used to prepare Yardstick test environment. For Euphrates, it supports: +Description: This API is used to prepare Yardstick test environment. +For Euphrates, it supports: -1. Prepare yardstick test environment, including set external network environment variable, load Yardstick VM images and create flavors; +1. Prepare yardstick test environment, including setting the + ``EXTERNAL_NETWORK`` environment variable, load Yardstick VM images and + create flavors; 2. Start an InfluxDB Docker container and config Yardstick output to InfluxDB; 3. Start a Grafana Docker container and config it with the InfluxDB. @@ -38,7 +42,8 @@ Example:: 'action': 'prepare_env' } -This is an asynchronous API. You need to call /yardstick/asynctask API to get the task result. +This is an asynchronous API. You need to call ``/yardstick/asynctask`` API to +get the task result. Start and config an InfluxDB docker container @@ -48,7 +53,8 @@ Example:: 'action': 'create_influxdb' } -This is an asynchronous API. You need to call /yardstick/asynctask API to get the task result. +This is an asynchronous API. You need to call ``/yardstick/asynctask`` API to +get the task result. Start and config a Grafana docker container @@ -58,11 +64,12 @@ Example:: 'action': 'create_grafana' } -This is an asynchronous API. You need to call /yardstick/asynctask API to get the task result. +This is an asynchronous API. You need to call ``/yardstick/asynctask`` API to +get the task result. /yardstick/asynctask -^^^^^^^^^^^^^^^^^^^^ +-------------------- Description: This API is used to get the status of asynchronous tasks @@ -84,7 +91,7 @@ NOTE:: /yardstick/testcases -^^^^^^^^^^^^^^^^^^^^ +-------------------- Description: This API is used to list all released Yardstick test cases. @@ -99,7 +106,7 @@ Example:: /yardstick/testcases/release/action -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------- Description: This API is used to run a Yardstick released test case. @@ -118,11 +125,12 @@ Example:: } } -This is an asynchronous API. You need to call /yardstick/results to get the result. +This is an asynchronous API. You need to call ``/yardstick/results`` to get the +result. /yardstick/testcases/samples/action -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------- Description: This API is used to run a Yardstick sample test case. @@ -141,13 +149,15 @@ Example:: } } -This is an asynchronous API. You need to call /yardstick/results to get the result. +This is an asynchronous API. You need to call ``/yardstick/results`` to get +the result. /yardstick/testcases/<testcase_name>/docs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------------- -Description: This API is used to the documentation of a certain released test case. +Description: This API is used to the documentation of a certain released test +case. Method: GET @@ -160,7 +170,7 @@ Example:: /yardstick/testsuites/action -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------- Description: This API is used to run a Yardstick test suite. @@ -179,11 +189,12 @@ Example:: } } -This is an asynchronous API. You need to call /yardstick/results to get the result. +This is an asynchronous API. You need to call /yardstick/results to get the +result. /yardstick/tasks/<task_id>/log -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------ Description: This API is used to get the real time log of test case execution. @@ -198,9 +209,11 @@ Example:: /yardstick/results -^^^^^^^^^^^^^^^^^^ +------------------ -Description: This API is used to get the test results of tasks. If you call /yardstick/testcases/samples/action API, it will return a task id. You can use the returned task id to get the results by using this API. +Description: This API is used to get the test results of tasks. If you call +/yardstick/testcases/samples/action API, it will return a task id. You can use +the returned task id to get the results by using this API. Method: GET @@ -215,9 +228,10 @@ This API will return a list of test case result /api/v2/yardstick/openrcs -^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------- -Description: This API provides functionality of handling OpenStack credential file (openrc). For Euphrates, it supports: +Description: This API provides functionality of handling OpenStack credential +file (openrc). For Euphrates, it supports: 1. Upload an openrc file for an OpenStack environment; 2. Update an openrc; @@ -268,7 +282,7 @@ Example:: /api/v2/yardstick/openrcs/<openrc_id> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------- Description: This API provides functionality of handling OpenStack credential file (openrc). For Euphrates, it supports: @@ -294,9 +308,10 @@ Example:: /api/v2/yardstick/pods -^^^^^^^^^^^^^^^^^^^^^^ +---------------------- -Description: This API provides functionality of handling Yardstick pod file (pod.yaml). For Euphrates, it supports: +Description: This API provides functionality of handling Yardstick pod file +(pod.yaml). For Euphrates, it supports: 1. Upload a pod file; @@ -319,7 +334,7 @@ Example:: /api/v2/yardstick/pods/<pod_id> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------- Description: This API provides functionality of handling Yardstick pod file (pod.yaml). For Euphrates, it supports: @@ -343,9 +358,10 @@ Example:: /api/v2/yardstick/images -^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------ -Description: This API is used to do some work related to Yardstick VM images. For Euphrates, it supports: +Description: This API is used to do some work related to Yardstick VM images. +For Euphrates, it supports: 1. Load Yardstick VM images; @@ -367,7 +383,7 @@ Example:: /api/v2/yardstick/images/<image_id> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------- Description: This API is used to do some work related to Yardstick VM images. For Euphrates, it supports: @@ -391,9 +407,10 @@ Example:: /api/v2/yardstick/tasks -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- -Description: This API is used to do some work related to yardstick tasks. For Euphrates, it supports: +Description: This API is used to do some work related to yardstick tasks. For +Euphrates, it supports: 1. Create a Yardstick task; @@ -416,7 +433,7 @@ Example:: /api/v2/yardstick/tasks/<task_id> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------- Description: This API is used to do some work related to yardstick tasks. For Euphrates, it supports: @@ -496,13 +513,15 @@ METHOD: DELETE Delete a task Example:: + http://<SERVER IP>:<PORT>/api/v2/yardstick/tasks/5g6g3e02-155a-4847-a5f8-154f1b31db8c /api/v2/yardstick/testcases -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------- -Description: This API is used to do some work related to yardstick testcases. For Euphrates, it supports: +Description: This API is used to do some work related to Yardstick testcases. +For Euphrates, it supports: 1. Upload a test case; 2. Get all released test cases' information; @@ -534,7 +553,7 @@ Example:: /api/v2/yardstick/testcases/<case_name> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------------------- Description: This API is used to do some work related to yardstick testcases. For Euphrates, it supports: @@ -555,13 +574,15 @@ METHOD: DELETE Delete a certain test case Example:: + http://<SERVER IP>:<PORT>/api/v2/yardstick/testcases/opnfv_yardstick_tc002 /api/v2/yardstick/testsuites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------- -Description: This API is used to do some work related to yardstick test suites. For Euphrates, it supports: +Description: This API is used to do some work related to yardstick test suites. +For Euphrates, it supports: 1. Create a test suite; 2. Get all test suites; @@ -596,7 +617,7 @@ Example:: /api/v2/yardstick/testsuites -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------- Description: This API is used to do some work related to yardstick test suites. For Euphrates, it supports: @@ -622,9 +643,10 @@ Example:: /api/v2/yardstick/projects -^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- -Description: This API is used to do some work related to yardstick test projects. For Euphrates, it supports: +Description: This API is used to do some work related to Yardstick test +projects. For Euphrates, it supports: 1. Create a Yardstick project; 2. Get all projects; @@ -656,7 +678,7 @@ Example:: /api/v2/yardstick/projects -^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- Description: This API is used to do some work related to yardstick test projects. For Euphrates, it supports: @@ -682,9 +704,10 @@ Example:: /api/v2/yardstick/containers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------- -Description: This API is used to do some work related to Docker containers. For Euphrates, it supports: +Description: This API is used to do some work related to Docker containers. +For Euphrates, it supports: 1. Create a Grafana Docker container; 2. Create an InfluxDB Docker container; @@ -721,7 +744,7 @@ Example:: /api/v2/yardstick/containers/<container_id> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------------- Description: This API is used to do some work related to Docker containers. For Euphrates, it supports: diff --git a/docs/testing/user/userguide/09-yardstick_user_interface.rst b/docs/testing/user/userguide/10-yardstick-user-interface.rst index 9058dd46d..cadec78ef 100644 --- a/docs/testing/user/userguide/09-yardstick_user_interface.rst +++ b/docs/testing/user/userguide/10-yardstick-user-interface.rst @@ -1,3 +1,4 @@ +======================== Yardstick User Interface ======================== @@ -6,14 +7,14 @@ in table format and also values pinned on to a graph. Command -------- +======= :: yardstick report generate <task-ID> <testcase-filename> Description ------------ +=========== 1. When the command is triggered using the task-id and the testcase name provided the respective values are retrieved from the diff --git a/docs/testing/user/userguide/10-vtc-overview.rst b/docs/testing/user/userguide/11-vtc-overview.rst index 8ed17873d..47582358c 100644 --- a/docs/testing/user/userguide/10-vtc-overview.rst +++ b/docs/testing/user/userguide/11-vtc-overview.rst @@ -29,10 +29,10 @@ to run the :term:`VNF`. The exploitation of Deep Packet Inspection assumptions: * third parties unaffiliated with either source or recipient are able to -inspect each IP packet’s payload + inspect each IP packet's payload -* the classifier knows the relevant syntax of each application’s packet -payloads (protocol signatures, data patterns, etc.). +* the classifier knows the relevant syntax of each application's packet + payloads (protocol signatures, data patterns, etc.). The proposed :term:`DPI` based approach will only use an indicative, small number of the initial packets from each flow in order to identify the content @@ -47,14 +47,14 @@ Concepts ======== * *Traffic Inspection*: The process of packet analysis and application -identification of network traffic that passes through the :term:`VTC`. + identification of network traffic that passes through the :term:`VTC`. * *Traffic Forwarding*: The process of packet forwarding from an incoming -network interface to a pre-defined outgoing network interface. + network interface to a pre-defined outgoing network interface. * *Traffic Rule Application*: The process of packet tagging, based on a -predefined set of rules. Packet tagging may include e.g. Type of Service -(:term:`ToS`) field modification. + predefined set of rules. Packet tagging may include e.g. Type of Service + (:term:`ToS`) field modification. Architecture ============ diff --git a/docs/testing/user/userguide/11-nsb-overview.rst b/docs/testing/user/userguide/12-nsb-overview.rst index 332dba47d..71a5c1130 100644 --- a/docs/testing/user/userguide/11-nsb-overview.rst +++ b/docs/testing/user/userguide/12-nsb-overview.rst @@ -3,11 +3,12 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, 2016-2017 Intel Corporation. +=================================== Network Services Benchmarking (NSB) =================================== Abstract --------- +======== .. _Yardstick: https://wiki.opnfv.org/yardstick @@ -15,10 +16,10 @@ This chapter provides an overview of the NSB, a contribution to OPNFV Yardstick_ from Intel. Overview --------- +======== -The goal of NSB is to Extend Yardstick to perform real world VNFs and NFVi Characterization and -benchmarking with repeatable and deterministic methods. +The goal of NSB is to Extend Yardstick to perform real world VNFs and NFVi +Characterization and benchmarking with repeatable and deterministic methods. The Network Service Benchmarking (NSB) extends the yardstick framework to do VNF characterization and benchmarking in three different execution @@ -70,17 +71,17 @@ NSB extension includes: - VNF KPIs, e.g., packet_in, packet_drop, packet_fwd etc Architecture ------------- +============ The Network Service (NS) defines a set of Virtual Network Functions (VNF) connected together using NFV infrastructure. The Yardstick NSB extension can support multiple VNFs created by different vendors including traffic generators. Every VNF being tested has its -own data model. The Network service defines a VNF modelling on base of performed -network functionality. The part of the data model is a set of the configuration -parameters, number of connection points used and flavor including core and -memory amount. +own data model. The Network service defines a VNF modelling on base of +performed network functionality. The part of the data model is a set of the +configuration parameters, number of connection points used and flavor including +core and memory amount. The ETSI defines a Network Service as a set of configurable VNFs working in some NFV Infrastructure connecting each other using Virtual Links available @@ -112,7 +113,7 @@ Network Service framework performs the necessary test steps. It may involve - Read the KPI's provided by particular VNF Components of Network Service -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------- * *Models for Network Service benchmarking*: The Network Service benchmarking requires the proper modelling approach. The NSB provides models using Python @@ -132,9 +133,9 @@ Components of Network Service same way as other VNFs being a part of benchmarked network service. Same as other VNFs the traffic generator are instantiated and terminated. - Every traffic generator has own configuration defined as a traffic profile and - a set of KPIs supported. The python models for TG is extended by specific calls - to listen and generate traffic. + Every traffic generator has own configuration defined as a traffic profile + and a set of KPIs supported. The python models for TG is extended by + specific calls to listen and generate traffic. * *The stateless TREX traffic generator*: The main traffic generator used as Network Service stimulus is open source TREX tool. @@ -165,7 +166,7 @@ Components of Network Service - RFC2544 throughput for various loss rate defined (1% is a default) Graphical Overview ------------------- +================== NSB Testing with yardstick framework facilitate performance testing of various VNFs provided. @@ -192,7 +193,7 @@ VNFs provided. Figure 1: Network Service - 2 server configuration VNFs supported for chracterization: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------- 1. CGNAPT - Carrier Grade Network Address and port Translation 2. vFW - Virtual Firewall diff --git a/docs/testing/user/userguide/12-nsb_installation.rst b/docs/testing/user/userguide/13-nsb-installation.rst index 5631c6578..1f6c79b0b 100644 --- a/docs/testing/user/userguide/12-nsb_installation.rst +++ b/docs/testing/user/userguide/13-nsb-installation.rst @@ -3,11 +3,12 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, 2016-2017 Intel Corporation. +===================================== Yardstick - NSB Testing -Installation ===================================== Abstract --------- +======== The Network Service Benchmarking (NSB) extends the yardstick framework to do VNF characterization and benchmarking in three different execution @@ -26,79 +27,65 @@ The steps needed to run Yardstick with NSB testing are: Prerequisites -------------- +============= Refer chapter Yardstick Installation for more information on yardstick prerequisites -Several prerequisites are needed for Yardstick(VNF testing): - - - Python Modules: pyzmq, pika. - - - flex - - - bison - - - build-essential - - - automake - - - libtool +Several prerequisites are needed for Yardstick (VNF testing): - - librabbitmq-dev - - - rabbitmq-server - - - collectd - - - intel-cmt-cat + * Python Modules: pyzmq, pika. + * flex + * bison + * build-essential + * automake + * libtool + * librabbitmq-dev + * rabbitmq-server + * collectd + * intel-cmt-cat Hardware & Software Ingredients -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------- SUT requirements: - +-----------+--------------------+ - | Item | Description | - +-----------+--------------------+ - | Memory | Min 20GB | - +-----------+--------------------+ - | NICs | 2 x 10G | - +-----------+--------------------+ - | OS | Ubuntu 16.04.3 LTS | - +-----------+--------------------+ - | kernel | 4.4.0-34-generic | - +-----------+--------------------+ - | DPDK | 17.02 | - +-----------+--------------------+ + ======= =================== + Item Description + ======= =================== + Memory Min 20GB + NICs 2 x 10G + OS Ubuntu 16.04.3 LTS + kernel 4.4.0-34-generic + DPDK 17.02 + ======= =================== Boot and BIOS settings: - +------------------+---------------------------------------------------+ - | Boot settings | default_hugepagesz=1G hugepagesz=1G hugepages=16 | - | | hugepagesz=2M hugepages=2048 isolcpus=1-11,22-33 | - | | nohz_full=1-11,22-33 rcu_nocbs=1-11,22-33 | - | | iommu=on iommu=pt intel_iommu=on | - | | Note: nohz_full and rcu_nocbs is to disable Linux | - | | kernel interrupts | - +------------------+---------------------------------------------------+ - |BIOS | CPU Power and Performance Policy <Performance> | - | | CPU C-state Disabled | - | | CPU P-state Disabled | - | | Enhanced Intel® Speedstep® Tech Disabled | - | | Hyper-Threading Technology (If supported) Enabled | - | | Virtualization Techology Enabled | - | | Intel(R) VT for Direct I/O Enabled | - | | Coherency Enabled | - | | Turbo Boost Disabled | - +------------------+---------------------------------------------------+ + ============= ================================================= + Boot settings default_hugepagesz=1G hugepagesz=1G hugepages=16 + hugepagesz=2M hugepages=2048 isolcpus=1-11,22-33 + nohz_full=1-11,22-33 rcu_nocbs=1-11,22-33 + iommu=on iommu=pt intel_iommu=on + Note: nohz_full and rcu_nocbs is to disable Linux + kernel interrupts + BIOS CPU Power and Performance Policy <Performance> + CPU C-state Disabled + CPU P-state Disabled + Enhanced Intel® Speedstep® Tech Disabl + Hyper-Threading Technology (If supported) Enabled + Virtualization Techology Enabled + Intel(R) VT for Direct I/O Enabled + Coherency Enabled + Turbo Boost Disabled + ============= ================================================= Install Yardstick (NSB Testing) -------------------------------- +=============================== Download the source code and install Yardstick from it @@ -148,6 +135,15 @@ Ansible: ansible_user=root ansible_pass=root +.. note:: + + SSH access without password needs to be configured for all your nodes defined in + ``yardstick-install-inventory.ini`` file. + If you want to use password authentication you need to install sshpass + + .. code-block:: console + + sudo -EH apt-get install sshpass To execute an installation for a Bare-Metal or a Standalone context: @@ -168,11 +164,12 @@ Above command setup docker with latest yardstick code. To execute docker exec -it yardstick bash -It will also automatically download all the packages needed for NSB Testing setup. -Refer chapter :doc:`04-installation` for more on docker **Install Yardstick using Docker (recommended)** +It will also automatically download all the packages needed for NSB Testing +setup. Refer chapter :doc:`04-installation` for more on docker +**Install Yardstick using Docker (recommended)** System Topology: ----------------- +================ .. code-block:: console @@ -187,13 +184,15 @@ System Topology: Environment parameters and credentials --------------------------------------- +====================================== Config yardstick conf -^^^^^^^^^^^^^^^^^^^^^ +--------------------- -If user did not run 'yardstick env influxdb' inside the container, which will generate -correct yardstick.conf, then create the config file manually (run inside the container): +If user did not run 'yardstick env influxdb' inside the container, which will +generate correct ``yardstick.conf``, then create the config file manually (run +inside the container): +:: cp ./etc/yardstick/yardstick.conf.sample /etc/yardstick/yardstick.conf vi /etc/yardstick/yardstick.conf @@ -219,11 +218,11 @@ Add trex_path, trex_client_lib and bin_path in 'nsb' section. trex_client_lib=/opt/nsb_bin/trex_client/stl Run Yardstick - Network Service Testcases ------------------------------------------ +========================================= NS testing - using yardstick CLI -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------- See :doc:`04-installation` @@ -236,13 +235,13 @@ NS testing - using yardstick CLI yardstick --debug task start yardstick/samples/vnf_samples/nsut/<vnf>/<test case> Network Service Benchmarking - Bare-Metal ------------------------------------------ +========================================= Bare-Metal Config pod.yaml describing Topology -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------------------------- -Bare-Metal 2-Node setup: -######################## +Bare-Metal 2-Node setup +^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: console +----------+ +----------+ @@ -254,8 +253,8 @@ Bare-Metal 2-Node setup: +----------+ +----------+ trafficgen_1 vnf -Bare-Metal 3-Node setup - Correlated Traffic: -############################################# +Bare-Metal 3-Node setup - Correlated Traffic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: console +----------+ +----------+ +------------+ @@ -270,7 +269,7 @@ Bare-Metal 3-Node setup - Correlated Traffic: Bare-Metal Config pod.yaml -^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- Before executing Yardstick test cases, make sure that pod.yaml reflects the topology and update all the required fields.:: @@ -345,13 +344,13 @@ topology and update all the required fields.:: Network Service Benchmarking - Standalone Virtualization --------------------------------------------------------- +======================================================== -SR-IOV: -^^^^^^^ +SR-IOV +------ SR-IOV Pre-requisites -##################### +^^^^^^^^^^^^^^^^^^^^^ On Host: a) Create a bridge for VM to connect to external network @@ -387,10 +386,10 @@ On Host: SR-IOV Config pod.yaml describing Topology -########################################## +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SR-IOV 2-Node setup: -#################### +^^^^^^^^^^^^^^^^^^^^ .. code-block:: console +--------------------+ @@ -418,7 +417,7 @@ SR-IOV 2-Node setup: SR-IOV 3-Node setup - Correlated Traffic -######################################## +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: console +--------------------+ @@ -454,7 +453,7 @@ topology and update all the required fields. .. note:: Update all the required fields like ip, user, password, pcis, etc... SR-IOV Config pod_trex.yaml -########################### +^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: YAML @@ -483,7 +482,7 @@ SR-IOV Config pod_trex.yaml local_mac: "00:00.00:00:00:02" SR-IOV Config host_sriov.yaml -############################# +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: YAML @@ -495,7 +494,8 @@ SR-IOV Config host_sriov.yaml user: "" password: "" -SR-IOV testcase update: ``<yardstick>/samples/vnf_samples/nsut/vfw/tc_sriov_rfc2544_ipv4_1rule_1flow_64B_trex.yaml`` +SR-IOV testcase update: +``<yardstick>/samples/vnf_samples/nsut/vfw/tc_sriov_rfc2544_ipv4_1rule_1flow_64B_trex.yaml`` Update "contexts" section """"""""""""""""""""""""" @@ -542,11 +542,11 @@ Update "contexts" section -OVS-DPDK: -^^^^^^^^^ +OVS-DPDK +-------- OVS-DPDK Pre-requisites -####################### +^^^^^^^^^^^^^^^^^^^^^^^ On Host: a) Create a bridge for VM to connect to external network @@ -585,10 +585,10 @@ On Host: OVS-DPDK Config pod.yaml describing Topology -############################################ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -OVS-DPDK 2-Node setup: -###################### +OVS-DPDK 2-Node setup +^^^^^^^^^^^^^^^^^^^^^ .. code-block:: console @@ -619,7 +619,7 @@ OVS-DPDK 2-Node setup: OVS-DPDK 3-Node setup - Correlated Traffic -########################################## +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: console @@ -659,7 +659,7 @@ topology and update all the required fields. .. note:: Update all the required fields like ip, user, password, pcis, etc... OVS-DPDK Config pod_trex.yaml -############################# +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: YAML @@ -687,7 +687,7 @@ OVS-DPDK Config pod_trex.yaml local_mac: "00:00.00:00:00:02" OVS-DPDK Config host_ovs.yaml -############################# +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: YAML @@ -699,7 +699,8 @@ OVS-DPDK Config host_ovs.yaml user: "" password: "" -ovs_dpdk testcase update: ``<yardstick>/samples/vnf_samples/nsut/vfw/tc_ovs_rfc2544_ipv4_1rule_1flow_64B_trex.yaml`` +ovs_dpdk testcase update: +``<yardstick>/samples/vnf_samples/nsut/vfw/tc_ovs_rfc2544_ipv4_1rule_1flow_64B_trex.yaml`` Update "contexts" section """"""""""""""""""""""""" @@ -757,7 +758,7 @@ Update "contexts" section Network Service Benchmarking - OpenStack with SR-IOV support ------------------------------------------------------------- +============================================================ This section describes how to run a Sample VNF test case, using Heat context, with SR-IOV. It also covers how to install OpenStack in Ubuntu 16.04, using @@ -765,7 +766,7 @@ DevStack, with SR-IOV support. Single node OpenStack setup with external TG -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------------------- .. code-block:: console @@ -796,7 +797,7 @@ Single node OpenStack setup with external TG Host pre-configuration -###################### +^^^^^^^^^^^^^^^^^^^^^^ .. warning:: The following configuration requires sudo access to the system. Make sure that your user have the access. @@ -896,7 +897,7 @@ Setup SR-IOV ports on the host: DevStack installation -##################### +^^^^^^^^^^^^^^^^^^^^^ Use official `Devstack <https://docs.openstack.org/devstack/pike/>`_ documentation to install OpenStack on a host. Please note, that stable @@ -918,7 +919,7 @@ Start the devstack installation on a host. TG host configuration -##################### +^^^^^^^^^^^^^^^^^^^^^ Yardstick automatically install and configure Trex traffic generator on TG host based on provided POD file (see below). Anyway, it's recommended to check @@ -927,7 +928,7 @@ the manual at https://trex-tgn.cisco.com/trex/doc/trex_manual.html. Run the Sample VNF test case -############################ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There is an example of Sample VNF test case ready to be executed in an OpenStack environment with SR-IOV support: ``samples/vnf_samples/nsut/vfw/ @@ -952,7 +953,7 @@ context using steps described in `NS testing - using yardstick CLI`_ section. Multi node OpenStack TG and VNF setup (two nodes) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------------------- .. code-block:: console @@ -983,14 +984,14 @@ Multi node OpenStack TG and VNF setup (two nodes) Controller/Compute pre-configuration -#################################### +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Pre-configuration of the controller and compute hosts are the same as described in `Host pre-configuration`_ section. Follow the steps in the section. DevStack configuration -###################### +^^^^^^^^^^^^^^^^^^^^^^ Use official `Devstack <https://docs.openstack.org/devstack/pike/>`_ documentation to install OpenStack on a host. Please note, that stable @@ -1017,7 +1018,7 @@ Start the devstack installation on the controller and compute hosts. Run the sample vFW TC -##################### +^^^^^^^^^^^^^^^^^^^^^ Install yardstick using `Install Yardstick (NSB Testing)`_ steps for OpenStack context. @@ -1034,30 +1035,31 @@ and the following yardtick command line arguments: Enabling other Traffic generator --------------------------------- +================================ -IxLoad: -^^^^^^^ +IxLoad +^^^^^^ -1. Software needed: IxLoadAPI ``<IxLoadTclApi verson>Linux64.bin.tgz and <IxOS - version>Linux64.bin.tar.gz`` (Download from ixia support site) - Install - ``<IxLoadTclApi verson>Linux64.bin.tgz & <IxOS version>Linux64.bin.tar.gz`` - If the installation was not done inside the container, after installing the IXIA client, - check /opt/ixia/ixload/<ver>/bin/ixloadpython and make sure you can run this cmd - inside the yardstick container. Usually user is required to copy or link /opt/ixia/python/<ver>/bin/ixiapython - to /usr/bin/ixiapython<ver> inside the container. +1. Software needed: IxLoadAPI ``<IxLoadTclApi verson>Linux64.bin.tgz`` and + ``<IxOS version>Linux64.bin.tar.gz`` (Download from ixia support site) + Install - ``<IxLoadTclApi verson>Linux64.bin.tgz`` and + ``<IxOS version>Linux64.bin.tar.gz`` + If the installation was not done inside the container, after installing + the IXIA client, check ``/opt/ixia/ixload/<ver>/bin/ixloadpython`` and make + sure you can run this cmd inside the yardstick container. Usually user is + required to copy or link ``/opt/ixia/python/<ver>/bin/ixiapython`` to + ``/usr/bin/ixiapython<ver>`` inside the container. -2. Update pod_ixia.yaml file with ixia details. +2. Update ``pod_ixia.yaml`` file with ixia details. .. code-block:: console cp <repo>/etc/yardstick/nodes/pod.yaml.nsb.sample.ixia etc/yardstick/nodes/pod_ixia.yaml - Config pod_ixia.yaml + Config ``pod_ixia.yaml`` .. code-block:: yaml - nodes: - name: trafficgen_1 @@ -1097,22 +1099,23 @@ IxLoad: You will also need to configure the IxLoad machine to start the IXIA IxosTclServer. This can be started like so: - - Connect to the IxLoad machine using RDP - - Go to: + * Connect to the IxLoad machine using RDP + * Go to: ``Start->Programs->Ixia->IxOS->IxOS 8.01-GA-Patch1->Ixia Tcl Server IxOS 8.01-GA-Patch1`` or ``"C:\Program Files (x86)\Ixia\IxOS\8.01-GA-Patch1\ixTclServer.exe"`` -4. Create a folder "Results" in c:\ and share the folder on the network. +4. Create a folder ``Results`` in c:\ and share the folder on the network. -5. execute testcase in samplevnf folder. - eg ``<repo>/samples/vnf_samples/nsut/vfw/tc_baremetal_http_ixload_1b_Requests-65000_Concurrency.yaml`` +5. Execute testcase in samplevnf folder e.g. + ``<repo>/samples/vnf_samples/nsut/vfw/tc_baremetal_http_ixload_1b_Requests-65000_Concurrency.yaml`` -IxNetwork: -^^^^^^^^^^ +IxNetwork +--------- -1. Software needed: ``IxNetworkAPI<ixnetwork verson>Linux64.bin.tgz`` (Download from ixia support site) - Install - ``IxNetworkAPI<ixnetwork verson>Linux64.bin.tgz`` +1. Software needed: ``IxNetworkAPI<ixnetwork verson>Linux64.bin.tgz`` + (Download from ixia support site) + Install - ``IxNetworkAPI<ixnetwork verson>Linux64.bin.tgz`` 2. Update pod_ixia.yaml file with ixia details. .. code-block:: console @@ -1162,9 +1165,11 @@ IxNetwork: You will also need to configure the IxNetwork machine to start the IXIA IxNetworkTclServer. This can be started like so: - - Connect to the IxNetwork machine using RDP - - Go to: ``Start->Programs->Ixia->IxNetwork->IxNetwork 7.21.893.14 GA->IxNetworkTclServer`` (or ``IxNetworkApiServer``) + * Connect to the IxNetwork machine using RDP + * Go to: + ``Start->Programs->Ixia->IxNetwork->IxNetwork 7.21.893.14 GA->IxNetworkTclServer`` + (or ``IxNetworkApiServer``) -4. execute testcase in samplevnf folder. - eg ``<repo>/samples/vnf_samples/nsut/vfw/tc_baremetal_rfc2544_ipv4_1rule_1flow_64B_ixia.yaml`` +4. Execute testcase in samplevnf folder e.g. + ``<repo>/samples/vnf_samples/nsut/vfw/tc_baremetal_rfc2544_ipv4_1rule_1flow_64B_ixia.yaml`` diff --git a/docs/testing/user/userguide/13-nsb_operation.rst b/docs/testing/user/userguide/14-nsb-operation.rst index e791b048d..2e741822e 100644 --- a/docs/testing/user/userguide/13-nsb_operation.rst +++ b/docs/testing/user/userguide/14-nsb-operation.rst @@ -23,13 +23,14 @@ provider/external networks. Provider networks ^^^^^^^^^^^^^^^^^ -The VNFs require a clear L2 connect to the external network in order to generate -realistic traffic from multiple address ranges and port +The VNFs require a clear L2 connect to the external network in order to +generate realistic traffic from multiple address ranges and ports. -In order to prevent Neutron from filtering traffic we have to disable Neutron Port Security. -We also disable DHCP on the data ports because we are binding the ports to DPDK and do not need -DHCP addresses. We also disable gateways because multiple default gateways can prevent SSH access -to the VNF from the floating IP. We only want a gateway on the mgmt network +In order to prevent Neutron from filtering traffic we have to disable Neutron +Port Security. We also disable DHCP on the data ports because we are binding +the ports to DPDK and do not need DHCP addresses. We also disable gateways +because multiple default gateways can prevent SSH access to the VNF from the +floating IP. We only want a gateway on the mgmt network .. code-block:: yaml @@ -42,8 +43,9 @@ to the VNF from the floating IP. We only want a gateway on the mgmt network Heat Topologies ^^^^^^^^^^^^^^^ -By default Heat will attach every node to every Neutron network that is created. -For scale-out tests we do not want to attach every node to every network. +By default Heat will attach every node to every Neutron network that is +created. For scale-out tests we do not want to attach every node to every +network. For each node you can specify which ports are on which network using the network_ports dictionary. @@ -85,11 +87,11 @@ In this example we have ``TRex xe0 <-> xe0 VNF xe1 <-> xe0 UDP_Replay`` Collectd KPIs ------------- -NSB can collect KPIs from collected. We have support for various plugins enabled by the -Barometer project. +NSB can collect KPIs from collected. We have support for various plugins +enabled by the Barometer project. -The default yardstick-samplevnf has collectd installed. This allows for collecting KPIs -from the VNF. +The default yardstick-samplevnf has collectd installed. This allows for +collecting KPIs from the VNF. Collecting KPIs from the NFVi is more complicated and requires manual setup. We assume that collectd is not installed on the compute nodes. @@ -130,15 +132,17 @@ Scale-Up VNFs performance data with scale-up - * Helps to figure out optimal number of cores specification in the Virtual Machine template creation or VNF + * Helps to figure out optimal number of cores specification in the Virtual + Machine template creation or VNF * Helps in comparison between different VNF vendor offerings - * Better the scale-up index, indicates the performance scalability of a particular solution + * Better the scale-up index, indicates the performance scalability of a + particular solution Heat ^^^^ - -For VNF scale-up tests we increase the number for VNF worker threads and ports. In the case of VNFs -we also need to increase the number of VCPUs and memory allocated to the VNF. +For VNF scale-up tests we increase the number for VNF worker threads. In the +case of VNFs we also need to increase the number of VCPUs and memory allocated +to the VNF. An example scale-up Heat testcase is: @@ -195,9 +199,9 @@ file may by specified in ``vnf_config`` scenario section. Baremetal ^^^^^^^^^ 1. Follow above traffic generator section to setup. - 2. edit num of threads in ``<repo>/samples/vnf_samples/nsut/vfw/tc_baremetal_rfc2544_ipv4_1rule_1flow_64B_trex_scale_up.yaml`` - - e.g, 6 Threads for given VNF + 2. Edit num of threads in + ``<repo>/samples/vnf_samples/nsut/vfw/tc_baremetal_rfc2544_ipv4_1rule_1flow_64B_trex_scale_up.yaml`` + e.g, 6 Threads for given VNF .. code-block:: yaml @@ -240,11 +244,12 @@ Baremetal Scale-Out -------------------- -VNFs performance data with scale-out +VNFs performance data with scale-out helps - * Helps in capacity planning to meet the given network node requirements - * Helps in comparison between different VNF vendor offerings - * Better the scale-out index, provides the flexibility in meeting future capacity requirements + * in capacity planning to meet the given network node requirements + * in comparison between different VNF vendor offerings + * better the scale-out index, provides the flexibility in meeting future + capacity requirements Standalone @@ -274,7 +279,8 @@ Scale-out not supported on Baremetal. Heat ^^^^ -There are sample scale-out all-VM Heat tests. These tests only use VMs and don't use external traffic. +There are sample scale-out all-VM Heat tests. These tests only use VMs and +don't use external traffic. The tests use UDP_Replay and correlated traffic. @@ -288,11 +294,14 @@ To run the test you need to increase OpenStack CPU, Memory and Port quotas. Traffic Generator tuning ------------------------ -The TRex traffic generator can be setup to use multiple threads per core, this is for multiqueue testing. +The TRex traffic generator can be setup to use multiple threads per core, this +is for multiqueue testing. -TRex does not automatically enable multiple threads because we currently cannot detect the number of queues on a device. +TRex does not automatically enable multiple threads because we currently cannot +detect the number of queues on a device. -To enable multiple queue set the queues_per_port value in the TG VNF options section. +To enable multiple queue set the ``queues_per_port`` value in the TG VNF +options section. .. code-block:: yaml diff --git a/docs/testing/user/userguide/15-list-of-tcs.rst b/docs/testing/user/userguide/15-list-of-tcs.rst index cb99c49cf..678f0f9a9 100644 --- a/docs/testing/user/userguide/15-list-of-tcs.rst +++ b/docs/testing/user/userguide/15-list-of-tcs.rst @@ -14,11 +14,11 @@ This chapter lists available Yardstick test cases. Yardstick test cases are divided in two main categories: * *Generic NFVI Test Cases* - Test Cases developed to realize the methodology -described in :doc:`02-methodology` + described in :doc:`02-methodology` * *OPNFV Feature Test Cases* - Test Cases developed to verify one or more -aspect of a feature delivered by an OPNFV Project, including the test cases -developed for the :term:`VTC`. + aspect of a feature delivered by an OPNFV Project, including the test cases + developed for the :term:`VTC`. Generic NFVI Test Case Descriptions =================================== @@ -109,8 +109,8 @@ Parser opnfv_yardstick_tc040.rst - StorPerf ------------ +StorPerf +-------- .. toctree:: :maxdepth: 1 diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 61e157e52..b936e723d 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -17,15 +17,16 @@ Yardstick User Guide 02-methodology 03-architecture 04-installation - 05-yardstick_plugin - 06-result-store-InfluxDB - 07-grafana - 08-api - 09-yardstick_user_interface - 10-vtc-overview - 11-nsb-overview - 12-nsb_installation - 13-nsb_operation + 05-operation + 06-yardstick-plugin + 07-result-store-InfluxDB + 08-grafana + 09-api + 10-yardstick-user-interface + 11-vtc-overview + 12-nsb-overview + 13-nsb-installation + 14-nsb-operation 15-list-of-tcs nsb/nsb-list-of-tcs glossary diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc050.rst b/docs/testing/user/userguide/opnfv_yardstick_tc050.rst index 8890c9d53..82a491b72 100644 --- a/docs/testing/user/userguide/opnfv_yardstick_tc050.rst +++ b/docs/testing/user/userguide/opnfv_yardstick_tc050.rst @@ -34,23 +34,20 @@ Yardstick Test Case Description TC050 | | 2) host: which is the name of a control node being attacked. | | | 3) interface: the network interface to be turned off. | | | | -| | There are four instance of the "close-interface" monitor: | -| | attacker1(for public netork): | -| | -fault_type: "close-interface" | -| | -host: node1 | -| | -interface: "br-ex" | -| | attacker2(for management netork): | -| | -fault_type: "close-interface" | -| | -host: node1 | -| | -interface: "br-mgmt" | -| | attacker3(for storage netork): | -| | -fault_type: "close-interface" | -| | -host: node1 | -| | -interface: "br-storage" | -| | attacker4(for private netork): | -| | -fault_type: "close-interface" | -| | -host: node1 | -| | -interface: "br-mesh" | +| | The interface to be closed by the attacker can be set by the | +| | variable of "{{ interface_name }}" | +| | | +| | attackers: | +| | - | +| | fault_type: "general-attacker" | +| | host: {{ attack_host }} | +| | key: "close-br-public" | +| | attack_key: "close-interface" | +| | action_parameter: | +| | interface: {{ interface_name }} | +| | rollback_parameter: | +| | interface: {{ interface_name }} | +| | | +--------------+--------------------------------------------------------------+ |monitors | In this test case, the monitor named "openstack-cmd" is | | | needed. The monitor needs needs two parameters: | @@ -61,17 +58,17 @@ Yardstick Test Case Description TC050 | | | | | There are four instance of the "openstack-cmd" monitor: | | | monitor1: | -| | -monitor_type: "openstack-cmd" | -| | -command_name: "nova image-list" | +| | - monitor_type: "openstack-cmd" | +| | - command_name: "nova image-list" | | | monitor2: | -| | -monitor_type: "openstack-cmd" | -| | -command_name: "neutron router-list" | +| | - monitor_type: "openstack-cmd" | +| | - command_name: "neutron router-list" | | | monitor3: | -| | -monitor_type: "openstack-cmd" | -| | -command_name: "heat stack-list" | +| | - monitor_type: "openstack-cmd" | +| | - command_name: "heat stack-list" | | | monitor4: | -| | -monitor_type: "openstack-cmd" | -| | -command_name: "cinder list" | +| | - monitor_type: "openstack-cmd" | +| | - command_name: "cinder list" | +--------------+--------------------------------------------------------------+ |metrics | In this test case, there is one metric: | | | 1)service_outage_time: which indicates the maximum outage | @@ -109,9 +106,9 @@ Yardstick Test Case Description TC050 +--------------+--------------------------------------------------------------+ |step 2 | do attacker: connect the host through SSH, and then execute | | | the turnoff network interface script with param value | -| | specified by "interface". | +| | specified by "{{ interface_name }}". | | | | -| | Result: Network interfaces will be turned down. | +| | Result: The specified network interface will be down. | | | | +--------------+--------------------------------------------------------------+ |step 3 | stop monitors after a period of time specified by | @@ -133,3 +130,4 @@ Yardstick Test Case Description TC050 | | execution problem. | | | | +--------------+--------------------------------------------------------------+ + diff --git a/install.sh b/install.sh index 6f4ab8c9d..04985f48a 100755 --- a/install.sh +++ b/install.sh @@ -84,7 +84,7 @@ apt-get update && apt-get install -y \ libxft-dev \ libxss-dev \ sudo \ - iputils-ping + iputils-ping \ rabbitmq-server if [[ "${DOCKER_ARCH}" != "aarch64" ]]; then @@ -95,10 +95,11 @@ apt-get -y autoremove && apt-get clean git config --global http.sslVerify false -# Configure and restart RabbitMQ +# Start and configure RabbitMQ +service rabbitmq-server restart +rabbitmqctl start_app rabbitmqctl add_user yardstick yardstick rabbitmqctl set_permissions yardstick ".*" ".*" ".*" -rabbitmqctl reset # install yardstick + dependencies easy_install -U pip==9.0.1 diff --git a/requirements.txt b/requirements.txt index 02545de1d..43f0e796c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,6 +37,7 @@ os-client-config==1.28.0 # OSI Approved Apache Software License osc-lib==1.7.0 # OSI Approved Apache Software License oslo.config==4.11.1 # OSI Approved Apache Software License oslo.i18n==3.17.0 # OSI Approved Apache Software License +oslo.messaging===5.30.2 # OSI Approved Apache Software License oslo.privsep===1.22.1 # OSI Approved Apache Software License oslo.serialization==2.20.1 # OSI Approved Apache Software License oslo.utils==3.28.0 # OSI Approved Apache Software License diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh index 5df769c0d..1e1591ce3 100755 --- a/tests/ci/load_images.sh +++ b/tests/ci/load_images.sh @@ -43,6 +43,12 @@ if [ "${YARD_IMG_ARCH}" == "arm64" ]; then fi fi +cleanup_loopbacks() { + # try again to cleanup loopbacks in case of error + losetup -a + losetup -O NAME,BACK-FILE | awk '/yardstick/ { print $1 }' | xargs -l1 losetup -v -d || true +} + build_yardstick_image() { echo @@ -56,6 +62,7 @@ build_yardstick_image() # Build the image. Retry once if the build fails $cmd || $cmd + cleanup_loopbacks if [ ! -f "${RAW_IMAGE}" ]; then echo "Failed building RAW image" exit 1 @@ -70,11 +77,25 @@ build_yardstick_image() -e YARD_IMG_ARCH=${YARD_IMG_ARCH} \ -vvv -i inventory.ini build_yardstick_image.yml + cleanup_loopbacks if [ ! -f "${QCOW_IMAGE}" ]; then echo "Failed building QCOW image" exit 1 fi fi + # DPDK compile is not enabled for arm64 yet so disable for now + # JIRA: YARSTICK-1124 + if [[ ! -f "${QCOW_NSB_IMAGE}" && ${DEPLOY_SCENARIO} == *[_-]ovs_dpdk[_-]* && "${YARD_IMG_ARCH}" != "arm64" ]]; then + ansible-playbook \ + -e img_property="nsb" \ + -e YARD_IMG_ARCH=${YARD_IMG_ARCH} \ + -vvv -i inventory.ini build_yardstick_image.yml + cleanup_loopbacks + if [ ! -f "${QCOW_NSB_IMAGE}" ]; then + echo "Failed building QCOW NSB image" + exit 1 + fi + fi fi } @@ -112,6 +133,18 @@ load_yardstick_image() ${EXTRA_PARAMS} \ --file ${QCOW_IMAGE} \ yardstick-image) + # DPDK compile is not enabled for arm64 yet so disable NSB images for now + # JIRA: YARSTICK-1124 + if [[ $DEPLOY_SCENARIO == *[_-]ovs_dpdk[_-]* && "${YARD_IMG_ARCH}" != "arm64" ]]; then + nsb_output=$(eval openstack ${SECURE} image create \ + --public \ + --disk-format qcow2 \ + --container-format bare \ + ${EXTRA_PARAMS} \ + --file ${QCOW_NSB_IMAGE} \ + yardstick-samplevnfs) + echo "$nsb_output" + fi fi echo "$output" @@ -232,6 +265,7 @@ create_nova_flavor() main() { QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-image.img" + QCOW_NSB_IMAGE="/tmp/workspace/yardstick/yardstick-nsb-image.img" RAW_IMAGE="/tmp/workspace/yardstick/yardstick-image.tar.gz" if [ -f /home/opnfv/images/yardstick-image.img ];then diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc050.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc050.yaml index dde3a1077..faddfab28 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc050.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc050.yaml @@ -13,12 +13,9 @@ description: > Yardstick TC050 config file; HA test case: OpenStack Controller Node Network High Availability. -{% set file = file or '/etc/yardstick/pod.yaml' %} {% set attack_host = attack_host or "node1" %} -{% set external_net = external_net or 'br-ex' %} -{% set management_net = management_net or 'br-mgmt' %} -{% set storage_net = storage_net or 'br-storage' %} -{% set internal_net = internal_net or 'br-mesh' %} +{% set interface_name = interface_name or 'br-mgmt' %} +{% set file = file or '/etc/yardstick/pod.yaml' %} scenarios: - @@ -27,43 +24,13 @@ scenarios: attackers: - fault_type: "general-attacker" - host: {{attack_host}} + host: {{ attack_host }} key: "close-br-public" attack_key: "close-interface" action_parameter: - interface: {{external_net}} - rollback_parameter: - interface: {{external_net}} - - - - fault_type: "general-attacker" - host: {{attack_host}} - key: "close-br-mgmt" - attack_key: "close-interface" - action_parameter: - interface: {{management_net}} - rollback_parameter: - interface: {{management_net}} - - - - fault_type: "general-attacker" - host: {{attack_host}} - key: "close-br-storage" - attack_key: "close-interface" - action_parameter: - interface: {{storage_net}} - rollback_parameter: - interface: {{storage_net}} - - - - fault_type: "general-attacker" - host: {{attack_host}} - key: "close-br-private" - attack_key: "close-interface" - action_parameter: - interface: {{internal_net}} + interface: {{ interface_name }} rollback_parameter: - interface: {{internal_net}} + interface: {{ interface_name }} monitors: - @@ -105,48 +72,34 @@ scenarios: steps: - - actionKey: "close-br-public" - actionType: "attacker" - index: 1 - - - - actionKey: "close-br-mgmt" - actionType: "attacker" - index: 2 - - - - actionKey: "close-br-storage" - actionType: "attacker" - index: 3 - - - - actionKey: "close-br-private" - actionType: "attacker" - index: 4 - - - actionKey: "nova-image-list" actionType: "monitor" - index: 5 + index: 1 - actionKey: "neutron-router-list" actionType: "monitor" - index: 6 + index: 2 - actionKey: "heat-stack-list" actionType: "monitor" - index: 7 + index: 3 - actionKey: "cinder-list" actionType: "monitor" - index: 8 + index: 4 + + - + actionKey: "close-br-public" + actionType: "attacker" + index: 5 + nodes: - {{attack_host}}: {{attack_host}}.LF + {{ attack_host }}: {{ attack_host }}.LF runner: type: Duration duration: 1 @@ -157,4 +110,4 @@ scenarios: context: type: Node name: LF - file: {{file}} + file: {{ file }} diff --git a/tests/opnfv/test_suites/opnfv_vTC_daily.yaml b/tests/opnfv/test_suites/opnfv_k8-canal-lb-noha_daily.yaml index f7efe51fb..80e0fbd39 100644 --- a/tests/opnfv/test_suites/opnfv_vTC_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_k8-canal-lb-noha_daily.yaml @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2017 Ericsson AB and others. +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,18 +7,12 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- -# ERICSSON POD1 VTC daily task suite +# k8 canal lb noha daily task suite schema: "yardstick:suite:0.1" -name: "opnfv_vTC_daily" +name: "k8-canal-lb-noha" test_cases_dir: "tests/opnfv/test_cases/" test_cases: - - file_name: opnfv_yardstick_tc006.yaml -- - file_name: opnfv_yardstick_tc007.yaml -- - file_name: opnfv_yardstick_tc020.yaml -- - file_name: opnfv_yardstick_tc021.yaml + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_vTC_weekly.yaml b/tests/opnfv/test_suites/opnfv_k8-multus-lb-noha_daily.yaml index 04f607ed4..6ec4d49c9 100644 --- a/tests/opnfv/test_suites/opnfv_vTC_weekly.yaml +++ b/tests/opnfv/test_suites/opnfv_k8-multus-lb-noha_daily.yaml @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2017 Ericsson AB and others. +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,18 +7,12 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- -# ERICSSON POD1 VTC weekly task suite +# k8 multus lb noha daily task suite schema: "yardstick:suite:0.1" -name: "opnfv_vTC_weekly" +name: "k8-multus-lb-noha" test_cases_dir: "tests/opnfv/test_cases/" test_cases: - - file_name: opnfv_yardstick_tc006.yaml -- - file_name: opnfv_yardstick_tc007.yaml -- - file_name: opnfv_yardstick_tc020.yaml -- - file_name: opnfv_yardstick_tc021.yaml + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_k8-multus-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_k8-multus-nofeature-noha_daily.yaml new file mode 100644 index 000000000..8b1ffeb7e --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_k8-multus-nofeature-noha_daily.yaml @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# k8 multus nofeature noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "k8-multus-nofeature-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-ha_daily.yaml new file mode 100644 index 000000000..d5386fd52 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-ha_daily.yaml @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# k8 nosdn nofeature ha daily task suite + +schema: "yardstick:suite:0.1" + +name: "k8-nosdn-nofeature-ha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-noha_daily.yaml index 4fd76596f..0ce94551a 100644 --- a/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_k8-nosdn-nofeature-noha_daily.yaml @@ -15,48 +15,4 @@ name: "k8-nosdn-nofeature-noha" test_cases_dir: "tests/opnfv/test_cases/" test_cases: - - file_name: opnfv_yardstick_tc002.yaml -- - file_name: opnfv_yardstick_tc005.yaml -- - file_name: opnfv_yardstick_tc010.yaml -- - file_name: opnfv_yardstick_tc011.yaml -- - file_name: opnfv_yardstick_tc012.yaml -- - file_name: opnfv_yardstick_tc014.yaml -- - file_name: opnfv_yardstick_tc037.yaml -- - file_name: opnfv_yardstick_tc055.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"file": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node5"}' -- - file_name: opnfv_yardstick_tc063.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"file": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node5"}' -- - file_name: opnfv_yardstick_tc069.yaml -- - file_name: opnfv_yardstick_tc070.yaml -- - file_name: opnfv_yardstick_tc071.yaml -- - file_name: opnfv_yardstick_tc072.yaml -- - file_name: opnfv_yardstick_tc075.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"file": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node1"}' + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_k8-ocl-lb-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_k8-ocl-lb-noha_daily.yaml new file mode 100644 index 000000000..a6ef9e1c9 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_k8-ocl-lb-noha_daily.yaml @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# k8 ocl lb noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "k8-ocl-lb-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_k8-sriov-cni-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_k8-sriov-cni-nofeature-noha_daily.yaml new file mode 100644 index 000000000..e0114f4b5 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_k8-sriov-cni-nofeature-noha_daily.yaml @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# k8 sriov-cni nofeature noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "k8-sriov-cni-nofeature-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc080.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-calipso-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-calipso-noha_daily.yaml new file mode 100644 index 000000000..3f0c4a992 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-calipso-noha_daily.yaml @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# os-nosdn-calipso-noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "os-nosdn-calipso-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc055.yaml + constraint: + installer: compass + pod: huawei-pod1 + task_args: + huawei-pod1: '{"file": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", + "host": "node5"}' +- + file_name: opnfv_yardstick_tc063.yaml + constraint: + installer: compass + pod: huawei-pod1 + task_args: + huawei-pod1: '{"file": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", + "host": "node5"}' +- + file_name: opnfv_yardstick_tc069.yaml +- + file_name: opnfv_yardstick_tc070.yaml +- + file_name: opnfv_yardstick_tc071.yaml +- + file_name: opnfv_yardstick_tc072.yaml +- + file_name: opnfv_yardstick_tc075.yaml + constraint: + installer: compass + pod: huawei-pod1 + task_args: + huawei-pod1: '{"file": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", + "host": "node1"}' diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml index ec0fd224c..bd91a75c7 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml @@ -21,18 +21,12 @@ test_cases: - file_name: opnfv_yardstick_tc006.yaml - - file_name: opnfv_yardstick_tc007.yaml -- file_name: opnfv_yardstick_tc008.yaml - file_name: opnfv_yardstick_tc009.yaml - file_name: opnfv_yardstick_tc011.yaml - - file_name: opnfv_yardstick_tc020.yaml -- - file_name: opnfv_yardstick_tc021.yaml -- file_name: opnfv_yardstick_tc037.yaml - file_name: opnfv_yardstick_tc038.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-ha_daily.yaml index ef47b9f72..201a271cc 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-ha_daily.yaml @@ -34,6 +34,11 @@ test_cases: - file_name: opnfv_yardstick_tc037.yaml - + file_name: opnfv_yardstick_tc042.yaml + constraint: + installer: compass + pod: huawei-pod1 +- file_name: opnfv_yardstick_tc055.yaml constraint: installer: compass diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-noha_daily.yaml index 2def5c2a6..e8db0de9b 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs_dpdk-noha_daily.yaml @@ -34,6 +34,11 @@ test_cases: - file_name: opnfv_yardstick_tc037.yaml - + file_name: opnfv_yardstick_tc042.yaml + constraint: + installer: compass + pod: huawei-pod1 +- file_name: opnfv_yardstick_tc055.yaml constraint: installer: compass diff --git a/tests/opnfv/test_suites/opnfv_os-odl-ovs_dpdk-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl-ovs_dpdk-ha_daily.yaml new file mode 100644 index 000000000..734d31d15 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-odl-ovs_dpdk-ha_daily.yaml @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# os-odl-ovs_dpdk-ha daily task suite + +schema: "yardstick:suite:0.1" + +name: "os-odl-ovs_dpdk-ha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc069.yaml +- + file_name: opnfv_yardstick_tc070.yaml +- + file_name: opnfv_yardstick_tc071.yaml +- + file_name: opnfv_yardstick_tc072.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-odl-ovs_dpdk-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl-ovs_dpdk-noha_daily.yaml new file mode 100644 index 000000000..e87dba74b --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-odl-ovs_dpdk-noha_daily.yaml @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# os-odl-ovs_dpdk-noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "os-odl-ovs_dpdk-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc069.yaml +- + file_name: opnfv_yardstick_tc070.yaml +- + file_name: opnfv_yardstick_tc071.yaml +- + file_name: opnfv_yardstick_tc072.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-odl-ovs_offload-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl-ovs_offload-noha_daily.yaml new file mode 100644 index 000000000..27d100a61 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-odl-ovs_offload-noha_daily.yaml @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# os-odl-ovs_offload-noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "os-odl-ovs_offload-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc069.yaml +- + file_name: opnfv_yardstick_tc070.yaml +- + file_name: opnfv_yardstick_tc071.yaml +- + file_name: opnfv_yardstick_tc072.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-odl-sriov-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl-sriov-noha_daily.yaml new file mode 100644 index 000000000..b6d0dd6af --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-odl-sriov-noha_daily.yaml @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# os-odl-sriov-noha daily task suite + +schema: "yardstick:suite:0.1" + +name: "os-odl-sriov-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc069.yaml +- + file_name: opnfv_yardstick_tc070.yaml +- + file_name: opnfv_yardstick_tc071.yaml +- + file_name: opnfv_yardstick_tc072.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml index 7172979c7..722d885b6 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml @@ -21,18 +21,12 @@ test_cases: - file_name: opnfv_yardstick_tc006.yaml - - file_name: opnfv_yardstick_tc007.yaml -- file_name: opnfv_yardstick_tc008.yaml - file_name: opnfv_yardstick_tc009.yaml - file_name: opnfv_yardstick_tc011.yaml - - file_name: opnfv_yardstick_tc020.yaml -- - file_name: opnfv_yardstick_tc021.yaml -- file_name: opnfv_yardstick_tc037.yaml - file_name: opnfv_yardstick_tc038.yaml diff --git a/tests/unit/network_services/helpers/test_samplevnf_helper.py b/tests/unit/network_services/helpers/test_samplevnf_helper.py index 3b6c89d3a..dc74b1859 100644 --- a/tests/unit/network_services/helpers/test_samplevnf_helper.py +++ b/tests/unit/network_services/helpers/test_samplevnf_helper.py @@ -143,8 +143,6 @@ class TestMultiPortConfig(unittest.TestCase): def setUp(self): self._mock_open = mock.patch.object(six.moves.builtins, 'open') self.mock_open = self._mock_open.start() - self._mock_os = mock.patch.object(os, 'path') - self.mock_os = self._mock_os.start() self._mock_config_parser = mock.patch.object( samplevnf_helper, 'ConfigParser') self.mock_config_parser = self._mock_config_parser.start() @@ -153,7 +151,6 @@ class TestMultiPortConfig(unittest.TestCase): def _cleanup(self): self._mock_open.stop() - self._mock_os.stop() self._mock_config_parser.stop() def test_validate_ip_and_prefixlen(self): @@ -185,7 +182,8 @@ class TestMultiPortConfig(unittest.TestCase): samplevnf_helper.MultiPortConfig.validate_ip_and_prefixlen( '::1', '129') - def test___init__(self): + @mock.patch.object(os.path, 'isfile', return_value=False) + def test___init__(self, *args): topology_file = mock.Mock() config_tpl = mock.Mock() tmp_file = mock.Mock() @@ -193,8 +191,6 @@ class TestMultiPortConfig(unittest.TestCase): opnfv_vnf = samplevnf_helper.MultiPortConfig( topology_file, config_tpl, tmp_file, vnfd_mock) self.assertEqual(0, opnfv_vnf.swq) - self.mock_os.path = mock.MagicMock() - self.mock_os.path.isfile = mock.Mock(return_value=False) opnfv_vnf = samplevnf_helper.MultiPortConfig( topology_file, config_tpl, tmp_file, vnfd_mock) self.assertEqual(0, opnfv_vnf.swq) diff --git a/tools/virt_ci_rampup.sh b/tools/virt_ci_rampup.sh index 6a9f2e7cb..d9aa23c22 100755 --- a/tools/virt_ci_rampup.sh +++ b/tools/virt_ci_rampup.sh @@ -15,7 +15,7 @@ ANSIBLE_SCRIPTS="${0%/*}/../ansible" -cd ${ANSIBLE_SCRIPTS} &&\ +cd ${ANSIBLE_SCRIPTS} && \ sudo -EH ansible-playbook \ - -e rs_file='../etc/infra/infra_deploy.yaml' \ + -e RS_FILE='../etc/infra/infra_deploy.yaml' -e CLEAN_UP=False \ -i inventory.ini infra_deploy.yml diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index 0d1dfb86f..0964b7baf 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -7,9 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import absolute_import -from __future__ import print_function - import collections import logging import os diff --git a/yardstick/benchmark/contexts/standalone/model.py b/yardstick/benchmark/contexts/standalone/model.py index f18d090d8..4d43f2611 100644 --- a/yardstick/benchmark/contexts/standalone/model.py +++ b/yardstick/benchmark/contexts/standalone/model.py @@ -232,14 +232,40 @@ class Libvirt(object): return ET.tostring(root) @staticmethod - def create_snapshot_qemu(connection, index, vm_image): - # build snapshot image - image = "/var/lib/libvirt/images/%s.qcow2" % index - connection.execute("rm %s" % image) - qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s" - connection.execute(qemu_template % (vm_image, image)) - - return image + def create_snapshot_qemu(connection, index, base_image): + """Create the snapshot image for a VM using a base image + + :param connection: SSH connection to the remote host + :param index: index of the VM to be spawn + :param base_image: path of the VM base image in the remote host + :return: snapshot image path + """ + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + connection.execute('rm -- "%s"' % vm_image) + status, _, _ = connection.execute('test -r %s' % base_image) + if status: + if not os.access(base_image, os.R_OK): + raise exceptions.LibvirtQemuImageBaseImageNotPresent( + vm_image=vm_image, base_image=base_image) + # NOTE(ralonsoh): done in two steps to avoid root permission + # issues. + LOG.info('Copy %s from execution host to remote host', base_image) + file_name = os.path.basename(os.path.normpath(base_image)) + connection.put_file(base_image, '/tmp/%s' % file_name) + status, _, error = connection.execute( + 'mv -- "/tmp/%s" "%s"' % (file_name, base_image)) + if status: + raise exceptions.LibvirtQemuImageCreateError( + vm_image=vm_image, base_image=base_image, error=error) + + LOG.info('Convert image %s to %s', base_image, vm_image) + qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + status, _, error = connection.execute(qemu_cmd) + if status: + raise exceptions.LibvirtQemuImageCreateError( + vm_image=vm_image, base_image=base_image, error=error) + return vm_image @classmethod def build_vm_xml(cls, connection, flavor, vm_name, index): diff --git a/yardstick/benchmark/contexts/standalone/ovs_dpdk.py b/yardstick/benchmark/contexts/standalone/ovs_dpdk.py index 30b685eec..b9e66a481 100644 --- a/yardstick/benchmark/contexts/standalone/ovs_dpdk.py +++ b/yardstick/benchmark/contexts/standalone/ovs_dpdk.py @@ -20,6 +20,7 @@ import re import time from yardstick import ssh +from yardstick.network_services.utils import get_nsb_option from yardstick.benchmark.contexts.base import Context from yardstick.benchmark.contexts.standalone import model from yardstick.common import exceptions @@ -55,7 +56,8 @@ class OvsDpdkContext(Context): self.file_path = None self.sriov = [] self.first_run = True - self.dpdk_devbind = '' + self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'), + 'dpdk-devbind.py') self.vm_names = [] self.nfvi_host = [] self.nodes = [] @@ -260,9 +262,6 @@ class OvsDpdkContext(Context): return self.connection = ssh.SSH.from_node(self.host_mgmt) - self.dpdk_devbind = utils.provision_tool( - self.connection, - os.path.join(utils.get_nsb_option('bin_path'), 'dpdk-devbind.py')) # Check dpdk/ovs version, if not present install self.check_ovs_dpdk_env() diff --git a/yardstick/benchmark/contexts/standalone/sriov.py b/yardstick/benchmark/contexts/standalone/sriov.py index 5db419e6a..95472fdda 100644 --- a/yardstick/benchmark/contexts/standalone/sriov.py +++ b/yardstick/benchmark/contexts/standalone/sriov.py @@ -19,7 +19,6 @@ import collections from yardstick import ssh from yardstick.network_services.utils import get_nsb_option -from yardstick.network_services.utils import provision_tool from yardstick.benchmark.contexts.base import Context from yardstick.benchmark.contexts.standalone import model from yardstick.network_services.utils import PciAddress @@ -38,7 +37,8 @@ class SriovContext(Context): self.file_path = None self.sriov = [] self.first_run = True - self.dpdk_devbind = '' + self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'), + 'dpdk-devbind.py') self.vm_names = [] self.nfvi_host = [] self.nodes = [] @@ -79,9 +79,6 @@ class SriovContext(Context): return self.connection = ssh.SSH.from_node(self.host_mgmt) - self.dpdk_devbind = provision_tool( - self.connection, - os.path.join(get_nsb_option("bin_path"), "dpdk-devbind.py")) # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config. model.StandaloneContextHelper.install_req_libs(self.connection) diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py index 99386a440..fbdf6c281 100755 --- a/yardstick/benchmark/runners/base.py +++ b/yardstick/benchmark/runners/base.py @@ -121,7 +121,7 @@ class Runner(object): @staticmethod def terminate_all(): """Terminate all runners (subprocesses)""" - log.debug("Terminating all runners", exc_info=True) + log.debug("Terminating all runners") # release dumper process as some errors before any runner is created if not Runner.runners: diff --git a/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash b/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash index 5a5dbc394..9f1804819 100644 --- a/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/unixbench_benchmark.bash @@ -18,7 +18,7 @@ OUTPUT_FILE=/tmp/unixbench-out.log # run unixbench test run_unixbench() { - cd /opt/tempT/UnixBench/ + cd /opt/tempT/UnixBench/UnixBench/ ./Run $OPTIONS > $OUTPUT_FILE } diff --git a/yardstick/benchmark/scenarios/lib/create_sec_group.py b/yardstick/benchmark/scenarios/lib/create_sec_group.py index 3d1aec9e8..1d2e36488 100644 --- a/yardstick/benchmark/scenarios/lib/create_sec_group.py +++ b/yardstick/benchmark/scenarios/lib/create_sec_group.py @@ -7,13 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import print_function -from __future__ import absolute_import - import logging from yardstick.benchmark.scenarios import base -import yardstick.common.openstack_utils as op_utils +from yardstick.common import openstack_utils +from yardstick.common import exceptions LOG = logging.getLogger(__name__) @@ -26,11 +24,12 @@ class CreateSecgroup(base.Scenario): def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg - self.options = self.scenario_cfg['options'] + self.options = self.scenario_cfg["options"] - self.sg_name = self.options.get("sg_name", "yardstick_sec_group") - self.description = self.options.get("description", None) - self.neutron_client = op_utils.get_neutron_client() + self.sg_name = self.options["sg_name"] + self.description = self.options.get("description", "") + self.project_id = self.options.get("project_id") + self.shade_client = openstack_utils.get_shade_client() self.setup_done = False @@ -45,21 +44,16 @@ class CreateSecgroup(base.Scenario): if not self.setup_done: self.setup() - sg_id = op_utils.create_security_group_full(self.neutron_client, - sg_name=self.sg_name, - sg_description=self.description) - - if sg_id: - result.update({"sg_create": 1}) - LOG.info("Create security group successful!") - else: + sg_id = openstack_utils.create_security_group_full( + self.shade_client, self.sg_name, sg_description=self.description, + project_id=self.project_id) + if not sg_id: result.update({"sg_create": 0}) LOG.error("Create security group failed!") + raise exceptions.ScenarioCreateSecurityGroupError - try: - keys = self.scenario_cfg.get('output', '').split() - except KeyError: - pass - else: - values = [sg_id] - return self._push_to_outputs(keys, values) + result.update({"sg_create": 1}) + LOG.info("Create security group successful!") + keys = self.scenario_cfg.get("output", '').split() + values = [sg_id] + return self._push_to_outputs(keys, values) diff --git a/yardstick/benchmark/scenarios/lib/delete_network.py b/yardstick/benchmark/scenarios/lib/delete_network.py index 2e8b595f9..8874e8b1e 100644 --- a/yardstick/benchmark/scenarios/lib/delete_network.py +++ b/yardstick/benchmark/scenarios/lib/delete_network.py @@ -10,7 +10,8 @@ import logging from yardstick.benchmark.scenarios import base -import yardstick.common.openstack_utils as op_utils +from yardstick.common import openstack_utils +from yardstick.common import exceptions LOG = logging.getLogger(__name__) @@ -24,11 +25,11 @@ class DeleteNetwork(base.Scenario): def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg - self.options = self.scenario_cfg['options'] + self.options = self.scenario_cfg["options"] - self.network_id = self.options.get("network_id", None) + self.network_name_or_id = self.options["network_name_or_id"] - self.shade_client = op_utils.get_shade_client() + self.shade_client = openstack_utils.get_shade_client() self.setup_done = False @@ -43,12 +44,13 @@ class DeleteNetwork(base.Scenario): if not self.setup_done: self.setup() - status = op_utils.delete_neutron_net(self.shade_client, - network_id=self.network_id) - if status: - result.update({"delete_network": 1}) - LOG.info("Delete network successful!") - else: + status = openstack_utils.delete_neutron_net(self.shade_client, + self.network_name_or_id) + + if not status: result.update({"delete_network": 0}) LOG.error("Delete network failed!") - return status + raise exceptions.ScenarioDeleteNetworkError + + result.update({"delete_network": 1}) + LOG.info("Delete network successful!") diff --git a/yardstick/cmd/commands/task.py b/yardstick/cmd/commands/task.py index a3488a23d..c6379e586 100644 --- a/yardstick/cmd/commands/task.py +++ b/yardstick/cmd/commands/task.py @@ -25,6 +25,7 @@ class TaskCommands(object): # pragma: no cover Set of commands to manage benchmark tasks. """ + EXIT_TEST_FAILED = 2 @cliargs("inputfile", type=str, help="path to task or suite file", nargs=1) @cliargs("--task-args", dest="task_args", @@ -48,18 +49,20 @@ class TaskCommands(object): # pragma: no cover param = change_osloobj_to_paras(args) self.output_file = param.output_file - result = {} LOG.info('Task START') try: result = Task().start(param, **kwargs) except Exception as e: # pylint: disable=broad-except self._write_error_data(e) - - if result.get('result', {}).get('criteria') == 'PASS': - LOG.info('Task SUCCESS') - else: LOG.info('Task FAILED') - raise RuntimeError('Task Failed') + raise + else: + if result.get('result', {}).get('criteria') == 'PASS': + LOG.info('Task SUCCESS') + else: + LOG.info('Task FAILED') + # exit without backtrace + raise SystemExit(self.EXIT_TEST_FAILED) def _write_error_data(self, error): data = {'status': 2, 'result': str(error)} diff --git a/yardstick/common/ansible_common.py b/yardstick/common/ansible_common.py index 38d2dd7c2..ca5a110e2 100644 --- a/yardstick/common/ansible_common.py +++ b/yardstick/common/ansible_common.py @@ -514,7 +514,7 @@ class AnsibleCommon(object): parser.add_section('defaults') parser.set('defaults', 'host_key_checking', 'False') - cfg_path = os.path.join(directory, 'setup.cfg') + cfg_path = os.path.join(directory, 'ansible.cfg') with open(cfg_path, 'w') as f: parser.write(f) diff --git a/yardstick/common/exceptions.py b/yardstick/common/exceptions.py index 439b9cb1b..ec21c335b 100644 --- a/yardstick/common/exceptions.py +++ b/yardstick/common/exceptions.py @@ -64,6 +64,11 @@ class YardstickBannedModuleImported(YardstickException): message = 'Module "%(module)s" cannnot be imported. Reason: "%(reason)s"' +class PayloadMissingAttributes(YardstickException): + message = ('Error instantiating a Payload class, missing attributes: ' + '%(missing_attributes)s') + + class HeatTemplateError(YardstickException): """Error in Heat during the stack deployment""" message = ('Error in Heat during the creation of the OpenStack stack ' @@ -112,6 +117,17 @@ class LibvirtCreateError(YardstickException): message = 'Error creating the virtual machine. Error: %(error)s.' +class LibvirtQemuImageBaseImageNotPresent(YardstickException): + message = ('Error creating the qemu image for %(vm_image)s. Base image: ' + '%(base_image)s. Base image not present in execution host or ' + 'remote host.') + + +class LibvirtQemuImageCreateError(YardstickException): + message = ('Error creating the qemu image for %(vm_image)s. Base image: ' + '%(base_image)s. Error: %(error)s.') + + class ScenarioConfigContextNameNotFound(YardstickException): message = 'Context name "%(context_name)s" not found' @@ -166,3 +182,11 @@ class ScenarioCreateFloatingIPError(YardstickException): class ScenarioDeleteFloatingIPError(YardstickException): message = 'Delete Neutron Floating IP Scenario failed' + + +class ScenarioCreateSecurityGroupError(YardstickException): + message = 'Create Neutron Security Group Scenario failed' + + +class ScenarioDeleteNetworkError(YardstickException): + message = 'Delete Neutron Network Scenario failed' diff --git a/yardstick/common/messaging/__init__.py b/yardstick/common/messaging/__init__.py new file mode 100644 index 000000000..f0f012ec3 --- /dev/null +++ b/yardstick/common/messaging/__init__.py @@ -0,0 +1,36 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MQ is statically configured: +# - MQ service: RabbitMQ +# - user/password: yardstick/yardstick +# - host:port: localhost:5672 +MQ_USER = 'yardstick' +MQ_PASS = 'yardstick' +MQ_SERVICE = 'rabbit' +SERVER = 'localhost' +PORT = 5672 +TRANSPORT_URL = (MQ_SERVICE + '://' + MQ_USER + ':' + MQ_PASS + '@' + SERVER + + ':' + str(PORT) + '/') + +# RPC server. +RPC_SERVER_EXECUTOR = 'threading' + +# Topics. +RUNNER = 'runner' + +# Methods. +# RUNNER methods: +RUNNER_INFO = 'runner_info' +RUNNER_LOOP = 'runner_loop' diff --git a/yardstick/common/messaging/consumer.py b/yardstick/common/messaging/consumer.py new file mode 100644 index 000000000..24ec6f184 --- /dev/null +++ b/yardstick/common/messaging/consumer.py @@ -0,0 +1,85 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import logging + +from oslo_config import cfg +import oslo_messaging +import six + +from yardstick.common import messaging + + +LOG = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class NotificationHandler(object): + """Abstract class to define a endpoint object for a MessagingConsumer""" + + def __init__(self, _id, ctx_pids, queue): + self._id = _id + self._ctx_pids = ctx_pids + self._queue = queue + + +@six.add_metaclass(abc.ABCMeta) +class MessagingConsumer(object): + """Abstract class to implement a MQ consumer + + This abstract class allows a class implementing this interface to receive + the messages published by a `MessagingNotifier`. + """ + + def __init__(self, topic, pids, endpoints, fanout=True): + """Init function. + + :param topic: (string) MQ exchange topic + :param pids: (list of int) list of PIDs of the processes implementing + the MQ Notifier which will be in the message context + :param endpoints: (list of class) list of classes implementing the + methods (see `MessagingNotifier.send_message) used by + the Notifier + :param fanout: (bool) MQ clients may request that a copy of the message + be delivered to all servers listening on a topic by + setting fanout to ``True``, rather than just one of them + :returns: `MessagingConsumer` class object + """ + + self._pids = pids + self._endpoints = endpoints + self._transport = oslo_messaging.get_rpc_transport( + cfg.CONF, url=messaging.TRANSPORT_URL) + self._target = oslo_messaging.Target(topic=topic, fanout=fanout, + server=messaging.SERVER) + self._server = oslo_messaging.get_rpc_server( + self._transport, self._target, self._endpoints, + executor=messaging.RPC_SERVER_EXECUTOR, + access_policy=oslo_messaging.DefaultRPCAccessPolicy) + + def start_rpc_server(self): + """Start the RPC server.""" + if self._server: + self._server.start() + + def stop_rpc_server(self): + """Stop the RPC server.""" + if self._server: + self._server.stop() + + def wait(self): + """Wait for message processing to complete (blocking).""" + if self._server: + self._server.wait() diff --git a/yardstick/common/messaging/payloads.py b/yardstick/common/messaging/payloads.py new file mode 100644 index 000000000..d29d79808 --- /dev/null +++ b/yardstick/common/messaging/payloads.py @@ -0,0 +1,53 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +import six + +from yardstick.common import exceptions + + +@six.add_metaclass(abc.ABCMeta) +class Payload(object): + """Base Payload class to transfer data through the MQ service""" + + REQUIRED_FIELDS = {'version'} + + def __init__(self, **kwargs): + """Init method + + :param kwargs: (dictionary) attributes and values of the object + :returns: Payload object + """ + + if not all(req_field in kwargs for req_field in self.REQUIRED_FIELDS): + _attrs = set(kwargs) - self.REQUIRED_FIELDS + missing_attributes = ', '.join(str(_attr) for _attr in _attrs) + raise exceptions.PayloadMissingAttributes( + missing_attributes=missing_attributes) + + for name, value in kwargs.items(): + setattr(self, name, value) + + self._fields = set(kwargs.keys()) + + def obj_to_dict(self): + """Returns a dictionary with the attributes of the object""" + return {field: getattr(self, field) for field in self._fields} + + @classmethod + def dict_to_obj(cls, _dict): + """Returns a Payload object built from the dictionary elements""" + return cls(**_dict) diff --git a/yardstick/common/messaging/producer.py b/yardstick/common/messaging/producer.py new file mode 100644 index 000000000..b6adc0c17 --- /dev/null +++ b/yardstick/common/messaging/producer.py @@ -0,0 +1,70 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import logging +import os + +from oslo_config import cfg +import oslo_messaging +import six + +from yardstick.common import messaging + + +LOG = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class MessagingProducer(object): + """Abstract class to implement a MQ producer + + This abstract class allows a class implementing this interface to publish + messages in a message queue. + """ + + def __init__(self, topic, pid=os.getpid(), fanout=True): + """Init function. + + :param topic: (string) MQ exchange topic + :param pid: (int) PID of the process implementing this MQ Notifier + :param fanout: (bool) MQ clients may request that a copy of the message + be delivered to all servers listening on a topic by + setting fanout to ``True``, rather than just one of them + :returns: `MessagingNotifier` class object + """ + self._topic = topic + self._pid = pid + self._fanout = fanout + self._transport = oslo_messaging.get_rpc_transport( + cfg.CONF, url=messaging.TRANSPORT_URL) + self._target = oslo_messaging.Target(topic=topic, fanout=fanout, + server=messaging.SERVER) + self._notifier = oslo_messaging.RPCClient(self._transport, + self._target) + + def send_message(self, method, payload): + """Send a cast message, that will invoke a method without blocking. + + The cast() method is used to invoke an RPC method that does not return + a value. cast() RPC requests may be broadcast to all Servers listening + on a given topic by setting the fanout Target property to ``True``. + + :param methos: (string) method name, that must be implemented in the + consumer endpoints + :param payload: (subclass `Payload`) payload content + """ + self._notifier.cast({'pid': self._pid}, + method, + **payload.obj_to_dict()) diff --git a/yardstick/common/openstack_utils.py b/yardstick/common/openstack_utils.py index 2785230c0..0d6afc54a 100644 --- a/yardstick/common/openstack_utils.py +++ b/yardstick/common/openstack_utils.py @@ -625,39 +625,6 @@ def delete_floating_ip(shade_client, floating_ip_id, retry=1): return False -def get_security_groups(neutron_client): # pragma: no cover - try: - security_groups = neutron_client.list_security_groups()[ - 'security_groups'] - return security_groups - except Exception: # pylint: disable=broad-except - log.error("Error [get_security_groups(neutron_client)]") - return None - - -def get_security_group_id(neutron_client, sg_name): # pragma: no cover - security_groups = get_security_groups(neutron_client) - id = '' - for sg in security_groups: - if sg['name'] == sg_name: - id = sg['id'] - break - return id - - -def create_security_group(neutron_client, sg_name, - sg_description): # pragma: no cover - json_body = {'security_group': {'name': sg_name, - 'description': sg_description}} - try: - secgroup = neutron_client.create_security_group(json_body) - return secgroup['security_group'] - except Exception: # pylint: disable=broad-except - log.error("Error [create_security_group(neutron_client, '%s', " - "'%s')]", sg_name, sg_description) - return None - - def create_security_group_rule(shade_client, secgroup_name_or_id, port_range_min=None, port_range_max=None, protocol=None, remote_ip_prefix=None, @@ -712,42 +679,52 @@ def create_security_group_rule(shade_client, secgroup_name_or_id, return False -def create_security_group_full(neutron_client, sg_name, - sg_description): # pragma: no cover - sg_id = get_security_group_id(neutron_client, sg_name) - if sg_id != '': - log.info("Using existing security group '%s'...", sg_name) - else: - log.info("Creating security group '%s'...", sg_name) - SECGROUP = create_security_group(neutron_client, - sg_name, - sg_description) - if not SECGROUP: - log.error("Failed to create the security group...") - return None +def create_security_group_full(shade_client, sg_name, + sg_description, project_id=None): + security_group = shade_client.get_security_group(sg_name) - sg_id = SECGROUP['id'] - - log.debug("Security group '%s' with ID=%s created successfully.", - SECGROUP['name'], sg_id) - - log.debug("Adding ICMP rules in security group '%s'...", sg_name) - if not create_security_group_rule(neutron_client, sg_id, - 'ingress', 'icmp'): - log.error("Failed to create the security group rule...") - return None - - log.debug("Adding SSH rules in security group '%s'...", sg_name) - if not create_security_group_rule( - neutron_client, sg_id, 'ingress', 'tcp', '22', '22'): - log.error("Failed to create the security group rule...") - return None - - if not create_security_group_rule( - neutron_client, sg_id, 'egress', 'tcp', '22', '22'): - log.error("Failed to create the security group rule...") - return None - return sg_id + if security_group: + log.info("Using existing security group '%s'...", sg_name) + return security_group['id'] + + log.info("Creating security group '%s'...", sg_name) + try: + security_group = shade_client.create_security_group( + sg_name, sg_description, project_id=project_id) + except (exc.OpenStackCloudException, + exc.OpenStackCloudUnavailableFeature) as op_exc: + log.error("Error [create_security_group(shade_client, %s, %s)]. " + "Exception message: %s", sg_name, sg_description, + op_exc.orig_message) + return + + log.debug("Security group '%s' with ID=%s created successfully.", + security_group['name'], security_group['id']) + + log.debug("Adding ICMP rules in security group '%s'...", sg_name) + if not create_security_group_rule(shade_client, security_group['id'], + direction='ingress', protocol='icmp'): + log.error("Failed to create the security group rule...") + shade_client.delete_security_group(sg_name) + return + + log.debug("Adding SSH rules in security group '%s'...", sg_name) + if not create_security_group_rule(shade_client, security_group['id'], + direction='ingress', protocol='tcp', + port_range_min='22', + port_range_max='22'): + log.error("Failed to create the security group rule...") + shade_client.delete_security_group(sg_name) + return + + if not create_security_group_rule(shade_client, security_group['id'], + direction='egress', protocol='tcp', + port_range_min='22', + port_range_max='22'): + log.error("Failed to create the security group rule...") + shade_client.delete_security_group(sg_name) + return + return security_group['id'] # ********************************************* @@ -797,6 +774,18 @@ def delete_image(glance_client, image_id): # pragma: no cover return True +def list_images(shade_client=None): + if shade_client is None: + shade_client = get_shade_client() + + try: + return shade_client.list_images() + except exc.OpenStackCloudException as o_exc: + log.error("Error [list_images(shade_client)]." + "Exception message, '%s'", o_exc.orig_message) + return False + + # ********************************************* # CINDER # ********************************************* diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 357f66be8..44cc92a7c 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -136,6 +136,11 @@ def source_env(env_file): p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE, shell=True) output = p.communicate()[0] + + # sometimes output type would be binary_type, and it don't have splitlines + # method, so we need to decode + if isinstance(output, six.binary_type): + output = encodeutils.safe_decode(output) env = dict(line.split('=', 1) for line in output.splitlines() if '=' in line) os.environ.update(env) return env diff --git a/yardstick/orchestrator/heat.py b/yardstick/orchestrator/heat.py index d69f86044..5afa4151e 100644 --- a/yardstick/orchestrator/heat.py +++ b/yardstick/orchestrator/heat.py @@ -15,6 +15,7 @@ import datetime import getpass import logging import pkg_resources +import pprint import socket import tempfile import time @@ -22,6 +23,7 @@ import time from oslo_serialization import jsonutils from oslo_utils import encodeutils import shade +from shade._heat import event_utils import yardstick.common.openstack_utils as op_utils from yardstick.common import exceptions @@ -63,6 +65,10 @@ class HeatStack(object): self._update_stack_tracking() + def get_failures(self): + return event_utils.get_events(self._cloud, self._stack.id, + event_args={'resource_status': 'FAILED'}) + def get(self): """Retrieves an existing stack from the target cloud @@ -625,6 +631,9 @@ name (i.e. %s). return stack if stack.status != self.HEAT_STATUS_COMPLETE: + for event in stack.get_failures(): + log.error("%s", event.resource_status_reason) + log.error(pprint.pformat(self._template)) raise exceptions.HeatTemplateError(stack_name=self.name) log.info("Creating stack '%s' DONE in %d secs", diff --git a/yardstick/tests/functional/common/messaging/__init__.py b/yardstick/tests/functional/common/messaging/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/functional/common/messaging/__init__.py diff --git a/yardstick/tests/functional/common/messaging/test_messaging.py b/yardstick/tests/functional/common/messaging/test_messaging.py new file mode 100644 index 000000000..99874343b --- /dev/null +++ b/yardstick/tests/functional/common/messaging/test_messaging.py @@ -0,0 +1,99 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +import time + +from yardstick.common.messaging import consumer +from yardstick.common.messaging import payloads +from yardstick.common.messaging import producer +from yardstick.tests.functional import base + + +TOPIC = 'topic_MQ' +METHOD_INFO = 'info' + + +class DummyPayload(payloads.Payload): + REQUIRED_FIELDS = {'version', 'data'} + + +class DummyEndpoint(consumer.NotificationHandler): + + def info(self, ctxt, **kwargs): + if ctxt['pid'] in self._ctx_pids: + self._queue.put('ID {}, data: {}, pid: {}'.format( + self._id, kwargs['data'], ctxt['pid'])) + + +class DummyConsumer(consumer.MessagingConsumer): + + def __init__(self, _id, ctx_pids, queue): + self._id = _id + endpoints = [DummyEndpoint(_id, ctx_pids, queue)] + super(DummyConsumer, self).__init__(TOPIC, ctx_pids, endpoints) + + +class DummyProducer(producer.MessagingProducer): + pass + + +def _run_consumer(_id, ctx_pids, queue): + _consumer = DummyConsumer(_id, ctx_pids, queue) + _consumer.start_rpc_server() + _consumer.wait() + + +class MessagingTestCase(base.BaseFunctionalTestCase): + + @staticmethod + def _terminate_consumers(num_consumers, processes): + for i in range(num_consumers): + processes[i].terminate() + + def test_run_five_consumers(self): + output_queue = multiprocessing.Queue() + num_consumers = 10 + ctx_1 = 100001 + ctx_2 = 100002 + producers = [DummyProducer(TOPIC, pid=ctx_1), + DummyProducer(TOPIC, pid=ctx_2)] + + processes = [] + for i in range(num_consumers): + processes.append(multiprocessing.Process( + name='consumer_{}'.format(i), + target=_run_consumer, + args=(i, [ctx_1, ctx_2], output_queue))) + processes[i].start() + self.addCleanup(self._terminate_consumers, num_consumers, processes) + + time.sleep(2) # Let consumers to create the listeners + for producer in producers: + for message in ['message 0', 'message 1']: + producer.send_message(METHOD_INFO, + DummyPayload(version=1, data=message)) + + time.sleep(2) # Let consumers attend the calls + output = [] + while not output_queue.empty(): + output.append(output_queue.get(True, 1)) + + self.assertEqual(num_consumers * 4, len(output)) + msg_template = 'ID {}, data: {}, pid: {}' + for i in range(num_consumers): + for ctx in [ctx_1, ctx_2]: + for message in ['message 0', 'message 1']: + msg = msg_template.format(i, message, ctx) + self.assertIn(msg, output) diff --git a/yardstick/tests/integration/dummy-scenario-heat-context.yaml b/yardstick/tests/integration/dummy-scenario-heat-context.yaml new file mode 100644 index 000000000..7c980b412 --- /dev/null +++ b/yardstick/tests/integration/dummy-scenario-heat-context.yaml @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2018 Intel +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# Sample Heat context config with Dummy context + +schema: "yardstick:task:0.1" + +scenarios: +- + type: Dummy + + runner: + type: Duration + duration: 5 + interval: 1 + +context: + name: {{ context_name }} + image: cirros-0.3.5 + flavor: cirros256 + user: cirros + + servers: + athena: + name: athena + ares: + name: ares + + networks: + test: + name: test diff --git a/yardstick/tests/unit/apiserver/resources/v2/__init__.py b/yardstick/tests/unit/apiserver/resources/v2/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/unit/apiserver/resources/v2/__init__.py diff --git a/yardstick/tests/unit/apiserver/resources/v2/test_images.py b/yardstick/tests/unit/apiserver/resources/v2/test_images.py new file mode 100644 index 000000000..ab131eec5 --- /dev/null +++ b/yardstick/tests/unit/apiserver/resources/v2/test_images.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import mock + +import unittest + +from yardstick.tests.unit.apiserver import APITestCase +from api.resources.v2.images import format_image_info + + +class V2ImagesTestCase(APITestCase): + @mock.patch('yardstick.common.openstack_utils.list_images') + @mock.patch('yardstick.common.utils.source_env') + def test_get(self, _, mock_list_images): + if self.app is None: + unittest.skip('host config error') + return + + single_image = mock.MagicMock() + single_image.name = 'yardstick-image' + single_image.size = 16384 + single_image.status = 'active' + single_image.updated_at = '2018-04-08' + + mock_list_images.return_value = [single_image] + url = 'api/v2/yardstick/images' + resp = self._get(url) + self.assertEqual(resp.get('status'), 1) + + +class FormatImageInfoTestCase(unittest.TestCase): + def test_format_image_info(self): + image = mock.MagicMock() + image.name = 'yardstick-image' + image.size = 1048576 + image.status = 'active' + image.updated_at = '2018-04-08' + + image_dict = format_image_info(image) + self.assertEqual(image_dict.get('size'), 1) diff --git a/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py b/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py index b1dcee209..72e684a68 100644 --- a/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py +++ b/yardstick/tests/unit/benchmark/contexts/standalone/test_model.py @@ -13,11 +13,11 @@ # limitations under the License. import copy -import mock import os -import unittest import uuid +import mock +import unittest from xml.etree import ElementTree from yardstick import ssh @@ -172,14 +172,70 @@ class ModelLibvirtTestCase(unittest.TestCase): interface_address.get('function')) def test_create_snapshot_qemu(self): - result = "/var/lib/libvirt/images/0.qcow2" - with mock.patch("yardstick.ssh.SSH") as ssh: - ssh_mock = mock.Mock(autospec=ssh.SSH) - ssh_mock.execute = \ - mock.Mock(return_value=(0, "a", "")) - ssh.return_value = ssh_mock - image = model.Libvirt.create_snapshot_qemu(ssh_mock, "0", "ubuntu.img") - self.assertEqual(image, result) + self.mock_ssh.execute = mock.Mock(return_value=(0, 0, 0)) + index = 1 + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + base_image = '/tmp/base_image' + + model.Libvirt.create_snapshot_qemu(self.mock_ssh, index, base_image) + self.mock_ssh.execute.assert_has_calls([ + mock.call('rm -- "%s"' % vm_image), + mock.call('test -r %s' % base_image), + mock.call('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + ]) + + @mock.patch.object(os.path, 'basename', return_value='base_image') + @mock.patch.object(os.path, 'normpath') + @mock.patch.object(os, 'access', return_value=True) + def test_create_snapshot_qemu_no_image_remote(self, + mock_os_access, mock_normpath, mock_basename): + self.mock_ssh.execute = mock.Mock( + side_effect=[(0, 0, 0), (1, 0, 0), (0, 0, 0), (0, 0, 0)]) + index = 1 + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + base_image = '/tmp/base_image' + mock_normpath.return_value = base_image + + model.Libvirt.create_snapshot_qemu(self.mock_ssh, index, base_image) + self.mock_ssh.execute.assert_has_calls([ + mock.call('rm -- "%s"' % vm_image), + mock.call('test -r %s' % base_image), + mock.call('mv -- "/tmp/%s" "%s"' % ('base_image', base_image)), + mock.call('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + ]) + mock_os_access.assert_called_once_with(base_image, os.R_OK) + mock_normpath.assert_called_once_with(base_image) + mock_basename.assert_has_calls([mock.call(base_image)]) + self.mock_ssh.put_file.assert_called_once_with(base_image, + '/tmp/base_image') + + @mock.patch.object(os, 'access', return_value=False) + def test_create_snapshot_qemu_no_image_local(self, mock_os_access): + self.mock_ssh.execute = mock.Mock(side_effect=[(0, 0, 0), (1, 0, 0)]) + base_image = '/tmp/base_image' + + with self.assertRaises(exceptions.LibvirtQemuImageBaseImageNotPresent): + model.Libvirt.create_snapshot_qemu(self.mock_ssh, 3, base_image) + mock_os_access.assert_called_once_with(base_image, os.R_OK) + + def test_create_snapshot_qemu_error_qemuimg_command(self): + self.mock_ssh.execute = mock.Mock( + side_effect=[(0, 0, 0), (0, 0, 0), (1, 0, 0)]) + index = 1 + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + base_image = '/tmp/base_image' + + with self.assertRaises(exceptions.LibvirtQemuImageCreateError): + model.Libvirt.create_snapshot_qemu(self.mock_ssh, index, + base_image) + self.mock_ssh.execute.assert_has_calls([ + mock.call('rm -- "%s"' % vm_image), + mock.call('test -r %s' % base_image), + mock.call('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + ]) @mock.patch.object(model.Libvirt, 'pin_vcpu_for_perf', return_value='4,5') @mock.patch.object(model.Libvirt, 'create_snapshot_qemu', @@ -422,7 +478,7 @@ class OvsDeployTestCase(unittest.TestCase): def setUp(self): self._mock_ssh = mock.patch.object(ssh, 'SSH') - self.mock_ssh = self._mock_ssh .start() + self.mock_ssh = self._mock_ssh.start() self.ovs_deploy = model.OvsDeploy(self.mock_ssh, '/tmp/dpdk-devbind.py', self.OVS_DETAILS) diff --git a/yardstick/tests/unit/benchmark/contexts/test_heat.py b/yardstick/tests/unit/benchmark/contexts/test_heat.py index 625f97bf4..1d491fe60 100644 --- a/yardstick/tests/unit/benchmark/contexts/test_heat.py +++ b/yardstick/tests/unit/benchmark/contexts/test_heat.py @@ -229,7 +229,7 @@ class HeatContextTestCase(unittest.TestCase): self.assertRaises(y_exc.HeatTemplateError, self.test_context.deploy) - mock_path_exists.assert_called_once() + mock_path_exists.assert_called() mock_resources_template.assert_called_once() @mock.patch.object(os.path, 'exists', return_value=False) @@ -254,7 +254,7 @@ class HeatContextTestCase(unittest.TestCase): 'yardstick/resources/files/yardstick_key-', self.test_context._name_task_id]) mock_genkeys.assert_called_once_with(key_filename) - mock_path_exists.assert_called_once_with(key_filename) + mock_path_exists.assert_any_call(key_filename) @mock.patch.object(heat, 'HeatTemplate') @mock.patch.object(os.path, 'exists', return_value=False) @@ -280,7 +280,7 @@ class HeatContextTestCase(unittest.TestCase): 'yardstick/resources/files/yardstick_key-', self.test_context._name]) mock_genkeys.assert_called_once_with(key_filename) - mock_path_exists.assert_called_once_with(key_filename) + mock_path_exists.assert_any_call(key_filename) @mock.patch.object(heat, 'HeatTemplate') @mock.patch.object(os.path, 'exists', return_value=False) @@ -296,7 +296,6 @@ class HeatContextTestCase(unittest.TestCase): self.test_context._flags.no_setup = True self.test_context.template_file = '/bar/baz/some-heat-file' self.test_context.get_neutron_info = mock.MagicMock() - self.test_context.deploy() mock_retrieve_stack.assert_called_once_with(self.test_context._name) @@ -306,7 +305,7 @@ class HeatContextTestCase(unittest.TestCase): 'yardstick/resources/files/yardstick_key-', self.test_context._name]) mock_genkeys.assert_called_once_with(key_filename) - mock_path_exists.assert_called_once_with(key_filename) + mock_path_exists.assert_any_call(key_filename) @mock.patch.object(heat, 'HeatTemplate', return_value='heat_template') @mock.patch.object(heat.HeatContext, '_add_resources_to_template') @@ -334,7 +333,7 @@ class HeatContextTestCase(unittest.TestCase): 'yardstick/resources/files/yardstick_key-', self.test_context._name_task_id]) mock_genkeys.assert_called_once_with(key_filename) - mock_path_exists.assert_called_with(key_filename) + mock_path_exists.assert_any_call(key_filename) mock_call_gen_keys = mock.call.gen_keys(key_filename) mock_call_add_resources = ( diff --git a/yardstick/tests/unit/benchmark/scenarios/lib/test_create_sec_group.py b/yardstick/tests/unit/benchmark/scenarios/lib/test_create_sec_group.py index 21158ab17..0477a49d4 100644 --- a/yardstick/tests/unit/benchmark/scenarios/lib/test_create_sec_group.py +++ b/yardstick/tests/unit/benchmark/scenarios/lib/test_create_sec_group.py @@ -6,25 +6,54 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## + +from oslo_utils import uuidutils import unittest import mock -from yardstick.benchmark.scenarios.lib.create_sec_group import CreateSecgroup - - -class CreateSecGroupTestCase(unittest.TestCase): - - @mock.patch('yardstick.common.openstack_utils.get_neutron_client') - @mock.patch('yardstick.common.openstack_utils.create_security_group_full') - def test_create_sec_group(self, mock_get_neutron_client, mock_create_security_group_full): - options = { - 'openstack_paras': { - 'sg_name': 'yardstick_sec_group', - 'description': 'security group for yardstick manual VM' - } - } - args = {"options": options} - obj = CreateSecgroup(args, {}) - obj.run({}) - mock_get_neutron_client.assert_called_once() - mock_create_security_group_full.assert_called_once() +from yardstick.common import openstack_utils +from yardstick.common import exceptions +from yardstick.benchmark.scenarios.lib import create_sec_group + + +class CreateSecurityGroupTestCase(unittest.TestCase): + + def setUp(self): + + self._mock_create_security_group_full = mock.patch.object( + openstack_utils, 'create_security_group_full') + self.mock_create_security_group_full = ( + self._mock_create_security_group_full.start()) + self._mock_get_shade_client = mock.patch.object( + openstack_utils, 'get_shade_client') + self.mock_get_shade_client = self._mock_get_shade_client.start() + self._mock_log = mock.patch.object(create_sec_group, 'LOG') + self.mock_log = self._mock_log.start() + self.args = {'options': {'sg_name': 'yardstick_sg'}} + self.result = {} + + self.csecgp_obj = create_sec_group.CreateSecgroup(self.args, mock.ANY) + self.addCleanup(self._stop_mock) + + def _stop_mock(self): + self._mock_create_security_group_full.stop() + self._mock_get_shade_client.stop() + self._mock_log.stop() + + def test_run(self): + _uuid = uuidutils.generate_uuid() + self.csecgp_obj.scenario_cfg = {'output': 'id'} + self.mock_create_security_group_full.return_value = _uuid + output = self.csecgp_obj.run(self.result) + self.assertEqual({'sg_create': 1}, self.result) + self.assertEqual({'id': _uuid}, output) + self.mock_log.info.asset_called_once_with( + 'Create security group successful!') + + def test_run_fail(self): + self.mock_create_security_group_full.return_value = None + with self.assertRaises(exceptions.ScenarioCreateSecurityGroupError): + self.csecgp_obj.run(self.result) + self.assertEqual({'sg_create': 0}, self.result) + self.mock_log.error.assert_called_once_with( + 'Create security group failed!') diff --git a/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py b/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py index aef99ee94..b6dbf4791 100644 --- a/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py +++ b/yardstick/tests/unit/benchmark/scenarios/lib/test_delete_network.py @@ -11,7 +11,8 @@ from oslo_utils import uuidutils import unittest import mock -import yardstick.common.openstack_utils as op_utils +from yardstick.common import openstack_utils +from yardstick.common import exceptions from yardstick.benchmark.scenarios.lib import delete_network @@ -19,16 +20,17 @@ class DeleteNetworkTestCase(unittest.TestCase): def setUp(self): self._mock_delete_neutron_net = mock.patch.object( - op_utils, 'delete_neutron_net') + openstack_utils, "delete_neutron_net") self.mock_delete_neutron_net = self._mock_delete_neutron_net.start() self._mock_get_shade_client = mock.patch.object( - op_utils, 'get_shade_client') + openstack_utils, "get_shade_client") self.mock_get_shade_client = self._mock_get_shade_client.start() - self._mock_log = mock.patch.object(delete_network, 'LOG') + self._mock_log = mock.patch.object(delete_network, "LOG") self.mock_log = self._mock_log.start() - _uuid = uuidutils.generate_uuid() - self.args = {'options': {'network_id': _uuid}} - self._del_obj = delete_network.DeleteNetwork(self.args, mock.ANY) + self.args = {"options": {"network_name_or_id": ( + uuidutils.generate_uuid())}} + self.result = {} + self.del_obj = delete_network.DeleteNetwork(self.args, mock.ANY) self.addCleanup(self._stop_mock) @@ -39,11 +41,14 @@ class DeleteNetworkTestCase(unittest.TestCase): def test_run(self): self.mock_delete_neutron_net.return_value = True - self.assertTrue(self._del_obj.run({})) + self.assertIsNone(self.del_obj.run(self.result)) + self.assertEqual({"delete_network": 1}, self.result) self.mock_log.info.assert_called_once_with( "Delete network successful!") def test_run_fail(self): self.mock_delete_neutron_net.return_value = False - self.assertFalse(self._del_obj.run({})) + with self.assertRaises(exceptions.ScenarioDeleteNetworkError): + self.del_obj.run(self.result) + self.assertEqual({"delete_network": 0}, self.result) self.mock_log.error.assert_called_once_with("Delete network failed!") diff --git a/yardstick/tests/unit/common/messaging/__init__.py b/yardstick/tests/unit/common/messaging/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/unit/common/messaging/__init__.py diff --git a/yardstick/tests/unit/common/messaging/test_consumer.py b/yardstick/tests/unit/common/messaging/test_consumer.py new file mode 100644 index 000000000..612dcaecd --- /dev/null +++ b/yardstick/tests/unit/common/messaging/test_consumer.py @@ -0,0 +1,54 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo_config import cfg +import oslo_messaging + +from yardstick.common import messaging +from yardstick.common.messaging import consumer +from yardstick.tests.unit import base as ut_base + + +class TestEndPoint(object): + def action_1(self): + pass + + +class _MessagingConsumer(consumer.MessagingConsumer): + pass + + +class MessagingConsumerTestCase(ut_base.BaseUnitTestCase): + + def test__init(self): + with mock.patch.object(oslo_messaging, 'get_rpc_server') as \ + mock_get_rpc_server, \ + mock.patch.object(oslo_messaging, 'get_rpc_transport') as \ + mock_get_rpc_transport, \ + mock.patch.object(oslo_messaging, 'Target') as \ + mock_Target: + mock_get_rpc_transport.return_value = 'test_rpc_transport' + mock_Target.return_value = 'test_Target' + + _MessagingConsumer('test_topic', 'test_pid', [TestEndPoint], + fanout=True) + mock_get_rpc_transport.assert_called_once_with( + cfg.CONF, url=messaging.TRANSPORT_URL) + mock_Target.assert_called_once_with( + topic='test_topic', fanout=True, server=messaging.SERVER) + mock_get_rpc_server.assert_called_once_with( + 'test_rpc_transport', 'test_Target', [TestEndPoint], + executor=messaging.RPC_SERVER_EXECUTOR, + access_policy=oslo_messaging.DefaultRPCAccessPolicy) diff --git a/yardstick/tests/unit/common/messaging/test_payloads.py b/yardstick/tests/unit/common/messaging/test_payloads.py new file mode 100644 index 000000000..00ec220c9 --- /dev/null +++ b/yardstick/tests/unit/common/messaging/test_payloads.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from yardstick.common import exceptions +from yardstick.common.messaging import payloads +from yardstick.tests.unit import base as ut_base + + +class _DummyPayload(payloads.Payload): + REQUIRED_FIELDS = {'version', 'key1', 'key2'} + + +class PayloadTestCase(ut_base.BaseUnitTestCase): + + def test__init(self): + payload = _DummyPayload(version=1, key1='value1', key2='value2') + self.assertEqual(1, payload.version) + self.assertEqual('value1', payload.key1) + self.assertEqual('value2', payload.key2) + self.assertEqual(3, len(payload._fields)) + + def test__init_missing_required_fields(self): + with self.assertRaises(exceptions.PayloadMissingAttributes): + _DummyPayload(key1='value1', key2='value2') + + def test_obj_to_dict(self): + payload = _DummyPayload(version=1, key1='value1', key2='value2') + payload_dict = payload.obj_to_dict() + self.assertEqual({'version': 1, 'key1': 'value1', 'key2': 'value2'}, + payload_dict) + + def test_dict_to_obj(self): + _dict = {'version': 2, 'key1': 'value100', 'key2': 'value200'} + payload = _DummyPayload.dict_to_obj(_dict) + self.assertEqual(set(_dict.keys()), payload._fields) diff --git a/yardstick/tests/unit/common/messaging/test_producer.py b/yardstick/tests/unit/common/messaging/test_producer.py new file mode 100644 index 000000000..0289689dc --- /dev/null +++ b/yardstick/tests/unit/common/messaging/test_producer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo_config import cfg +import oslo_messaging + +from yardstick.common import messaging +from yardstick.common.messaging import producer +from yardstick.tests.unit import base as ut_base + + +class _MessagingProducer(producer.MessagingProducer): + pass + + +class MessagingProducerTestCase(ut_base.BaseUnitTestCase): + + def test__init(self): + with mock.patch.object(oslo_messaging, 'RPCClient') as \ + mock_RPCClient, \ + mock.patch.object(oslo_messaging, 'get_rpc_transport') as \ + mock_get_rpc_transport, \ + mock.patch.object(oslo_messaging, 'Target') as \ + mock_Target: + mock_get_rpc_transport.return_value = 'test_rpc_transport' + mock_Target.return_value = 'test_Target' + + _MessagingProducer('test_topic', 'test_pid', fanout=True) + mock_get_rpc_transport.assert_called_once_with( + cfg.CONF, url=messaging.TRANSPORT_URL) + mock_Target.assert_called_once_with( + topic='test_topic', fanout=True, server=messaging.SERVER) + mock_RPCClient.assert_called_once_with('test_rpc_transport', + 'test_Target') diff --git a/yardstick/tests/unit/common/test_openstack_utils.py b/yardstick/tests/unit/common/test_openstack_utils.py index 3b7e8eaa1..f03f2516c 100644 --- a/yardstick/tests/unit/common/test_openstack_utils.py +++ b/yardstick/tests/unit/common/test_openstack_utils.py @@ -39,18 +39,17 @@ class DeleteNeutronNetTestCase(unittest.TestCase): def setUp(self): self.mock_shade_client = mock.Mock() - self.mock_shade_client.delete_network = mock.Mock() def test_delete_neutron_net(self): self.mock_shade_client.delete_network.return_value = True output = openstack_utils.delete_neutron_net(self.mock_shade_client, - 'network_id') + 'network_name_or_id') self.assertTrue(output) def test_delete_neutron_net_fail(self): self.mock_shade_client.delete_network.return_value = False output = openstack_utils.delete_neutron_net(self.mock_shade_client, - 'network_id') + 'network_name_or_id') self.assertFalse(output) @mock.patch.object(openstack_utils, 'log') @@ -58,7 +57,7 @@ class DeleteNeutronNetTestCase(unittest.TestCase): self.mock_shade_client.delete_network.side_effect = ( exc.OpenStackCloudException('error message')) output = openstack_utils.delete_neutron_net(self.mock_shade_client, - 'network_id') + 'network_name_or_id') self.assertFalse(output) mock_logger.error.assert_called_once() @@ -264,3 +263,77 @@ class CreateSecurityGroupRuleTestCase(unittest.TestCase): self.mock_shade_client, self.secgroup_name_or_id) mock_logger.error.assert_called_once() self.assertFalse(output) + + +class ListImageTestCase(unittest.TestCase): + + def test_list_images(self): + mock_shade_client = mock.MagicMock() + mock_shade_client.list_images.return_value = [] + openstack_utils.list_images(mock_shade_client) + + @mock.patch.object(openstack_utils, 'log') + def test_list_images_exception(self, mock_logger): + mock_shade_client = mock.MagicMock() + mock_shade_client.list_images = mock.MagicMock() + mock_shade_client.list_images.side_effect = ( + exc.OpenStackCloudException('error message')) + images = openstack_utils.list_images(mock_shade_client) + mock_logger.error.assert_called_once() + self.assertFalse(images) + + +class SecurityGroupTestCase(unittest.TestCase): + + def setUp(self): + self.mock_shade_client = mock.Mock() + self.sg_name = 'sg_name' + self.sg_description = 'sg_description' + self._uuid = uuidutils.generate_uuid() + + def test_create_security_group_full_existing_security_group(self): + self.mock_shade_client.get_security_group.return_value = ( + {'name': 'name', 'id': self._uuid}) + output = openstack_utils.create_security_group_full( + self.mock_shade_client, self.sg_name, self.sg_description) + self.mock_shade_client.get_security_group.assert_called_once() + self.assertEqual(self._uuid, output) + + @mock.patch.object(openstack_utils, 'log') + def test_create_security_group_full_non_existing_security_group( + self, mock_logger): + self.mock_shade_client.get_security_group.return_value = None + self.mock_shade_client.create_security_group.side_effect = ( + exc.OpenStackCloudException('error message')) + output = openstack_utils.create_security_group_full( + self.mock_shade_client, self.sg_name, self.sg_description) + mock_logger.error.assert_called_once() + self.assertIsNone(output) + + @mock.patch.object(openstack_utils, 'create_security_group_rule') + @mock.patch.object(openstack_utils, 'log') + def test_create_security_group_full_create_rule_fail( + self, mock_logger, mock_create_security_group_rule): + self.mock_shade_client.get_security_group.return_value = None + self.mock_shade_client.create_security_group.return_value = ( + {'name': 'name', 'id': self._uuid}) + mock_create_security_group_rule.return_value = False + output = openstack_utils.create_security_group_full( + self.mock_shade_client, self.sg_name, self.sg_description) + mock_create_security_group_rule.assert_called() + self.mock_shade_client.delete_security_group(self.sg_name) + mock_logger.error.assert_called_once() + self.assertIsNone(output) + + @mock.patch.object(openstack_utils, 'create_security_group_rule') + def test_create_security_group_full( + self, mock_create_security_group_rule): + self.mock_shade_client.get_security_group.return_value = None + self.mock_shade_client.create_security_group.return_value = ( + {'name': 'name', 'id': self._uuid}) + mock_create_security_group_rule.return_value = True + output = openstack_utils.create_security_group_full( + self.mock_shade_client, self.sg_name, self.sg_description) + mock_create_security_group_rule.assert_called() + self.mock_shade_client.delete_security_group(self.sg_name) + self.assertEqual(self._uuid, output) diff --git a/yardstick/tests/unit/orchestrator/test_heat.py b/yardstick/tests/unit/orchestrator/test_heat.py index aae2487aa..9598eeb04 100644 --- a/yardstick/tests/unit/orchestrator/test_heat.py +++ b/yardstick/tests/unit/orchestrator/test_heat.py @@ -354,13 +354,30 @@ class HeatTemplateTestCase(unittest.TestCase): 3600) self.assertEqual(heat_stack, ret) - def test_create_block_status_no_complete(self): heat_stack = mock.Mock() heat_stack.status = 'other status' + heat_stack.get_failures.return_value = [] with mock.patch.object(heat, 'HeatStack', return_value=heat_stack): self.assertRaises(exceptions.HeatTemplateError, self.template.create, block=True) heat_stack.create.assert_called_once_with( self.template._template, self.template.heat_parameters, True, 3600) + + def test_create_block_status_no_complete_with_reasons(self): + heat_stack = mock.Mock() + heat_stack.status = 'other status' + heat_stack.get_failures.return_value = [ + mock.Mock(resource_status_reason="A reason"), + mock.Mock(resource_status_reason="Something else") + ] + with mock.patch.object(heat, 'HeatStack', return_value=heat_stack): + with mock.patch.object(heat, 'log') as mock_log: + self.assertRaises(exceptions.HeatTemplateError, + self.template.create, block=True) + mock_log.error.assert_any_call("%s", "A reason") + mock_log.error.assert_any_call("%s", "Something else") + heat_stack.create.assert_called_once_with( + self.template._template, self.template.heat_parameters, True, + 3600) |