diff options
24 files changed, 870 insertions, 310 deletions
diff --git a/api/api-prepare.sh b/api/api-prepare.sh index fade8ccc6..5cc65c959 100755 --- a/api/api-prepare.sh +++ b/api/api-prepare.sh @@ -20,7 +20,7 @@ server { index index.htm index.html; location / { include uwsgi_params; - uwsgi_pass unix:///home/opnfv/repos/yardstick/api/yardstick.sock; + uwsgi_pass unix:///var/run/yardstick.sock; } } EOF @@ -47,3 +47,9 @@ command = uwsgi -i yardstick.ini autorestart = true EOF fi + +# create api log directory +mkdir -p /var/log/yardstick + +# create yardstick.sock for communicating +touch /var/run/yardstick.sock diff --git a/api/database/__init__.py b/api/database/__init__.py new file mode 100644 index 000000000..bc2708bc7 --- /dev/null +++ b/api/database/__init__.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging + +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +engine = create_engine('sqlite:////tmp/yardstick.db', convert_unicode=True) +db_session = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=engine)) +Base = declarative_base() +Base.query = db_session.query_property() + + +def init_db(): + subclasses = [subclass.__name__ for subclass in Base.__subclasses__()] + logger.debug('Import models: %s', subclasses) + Base.metadata.create_all(bind=engine) diff --git a/api/database/models.py b/api/database/models.py new file mode 100644 index 000000000..25e323842 --- /dev/null +++ b/api/database/models.py @@ -0,0 +1,25 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from sqlalchemy import Column +from sqlalchemy import Integer +from sqlalchemy import String + +from api.database import Base + + +class Tasks(Base): + __tablename__ = 'tasks' + id = Column(Integer, primary_key=True) + task_id = Column(String(30)) + status = Column(Integer) + error = Column(String(120)) + details = Column(String(120)) + + def __repr__(self): + return '<Task %r>' % Tasks.task_id diff --git a/api/resources/env_action.py b/api/resources/env_action.py index fa0f95d90..59a1692a1 100644 --- a/api/resources/env_action.py +++ b/api/resources/env_action.py @@ -13,6 +13,7 @@ import time import json import os import errno +import ConfigParser from docker import Client @@ -104,7 +105,7 @@ def _create_influxdb(): client = Client(base_url=config.DOCKER_URL) try: - _config_output_file() + _change_output_to_influxdb() if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE, config.INFLUXDB_TAG)): @@ -144,29 +145,18 @@ def _config_influxdb(): logger.debug('Failed to config influxDB: %s', e) -def _config_output_file(): +def _change_output_to_influxdb(): yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) - with open(config.YARDSTICK_CONFIG_FILE, 'w') as f: - f.write("""\ -[DEFAULT] -debug = False -dispatcher = influxdb -[dispatcher_file] -file_path = /tmp/yardstick.out + parser = ConfigParser.ConfigParser() + parser.read(config.YARDSTICK_CONFIG_SAMPLE_FILE) -[dispatcher_http] -timeout = 5 -# target = http://127.0.0.1:8000/results + parser.set('DEFAULT', 'dispatcher', 'influxdb') + parser.set('dispatcher_influxdb', 'target', + 'http://%s:8086' % api_conf.GATEWAY_IP) -[dispatcher_influxdb] -timeout = 5 -target = http://%s:8086 -db_name = yardstick -username = root -password = root -""" - % api_conf.GATEWAY_IP) + with open(config.YARDSTICK_CONFIG_FILE, 'w') as f: + parser.write(f) def prepareYardstickEnv(args): @@ -195,6 +185,8 @@ def _prepare_env_daemon(): # update the external_network _source_file(rc_file) + _clean_images() + _load_images() @@ -251,6 +243,14 @@ def _append_external_network(rc_file): raise +def _clean_images(): + cmd = [config.CLEAN_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + cwd=config.YARDSTICK_REPOS_DIR) + output = p.communicate()[0] + logger.debug('The result is: %s', output) + + def _load_images(): cmd = [config.LOAD_IMAGES_SCRIPT] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, diff --git a/api/server.py b/api/server.py index 64a2b4f96..fac821b00 100644 --- a/api/server.py +++ b/api/server.py @@ -12,6 +12,8 @@ from flask import Flask from flask_restful import Api from flasgger import Swagger +from api.database import init_db +from api.database import db_session from api.urls import urlpatterns from yardstick import _init_logging @@ -19,11 +21,18 @@ logger = logging.getLogger(__name__) app = Flask(__name__) +init_db() + Swagger(app) api = Api(app) +@app.teardown_request +def shutdown_session(exception=None): + db_session.remove() + + reduce(lambda a, b: a.add_resource(b.resource, b.url, endpoint=b.endpoint) or a, urlpatterns, api) diff --git a/api/yardstick.ini b/api/yardstick.ini index 01025c2ef..2ba881fc1 100644 --- a/api/yardstick.ini +++ b/api/yardstick.ini @@ -12,5 +12,5 @@ chmod-socket = 666 callable = app enable-threads = true close-on-exec = 1 -daemonize=/home/opnfv/repos/yardstick/api/uwsgi.log -socket = /home/opnfv/repos/yardstick/api/yardstick.sock +daemonize= /var/log/yardstick/uwsgi.log +socket = /var/run/yardstick.sock diff --git a/api/yardstick.sock b/api/yardstick.sock deleted file mode 100644 index e69de29bb..000000000 --- a/api/yardstick.sock +++ /dev/null diff --git a/docs/userguide/03-architecture.rst b/docs/userguide/03-architecture.rst index ace3117c2..03bf00f58 100755 --- a/docs/userguide/03-architecture.rst +++ b/docs/userguide/03-architecture.rst @@ -175,7 +175,7 @@ LmBench, ...) TaskCommands is the "yardstick task" subcommand's main entry. It takes yaml file (e.g. test.yaml) as input, and uses HeatContext to convert the yaml -file's context section to HOT. After Openstacik heat stack is deployed by +file's context section to HOT. After Openstack heat stack is deployed by HeatContext with the converted HOT, TaskCommands use Runner to run specified TestScenario. During first runner initialization, it will create output process. The output process use Dispatcher to push test results. The Runner diff --git a/docs/userguide/opnfv_yardstick_tc001.rst b/docs/userguide/opnfv_yardstick_tc001.rst index fac375d50..b53c508a6 100644 --- a/docs/userguide/opnfv_yardstick_tc001.rst +++ b/docs/userguide/opnfv_yardstick_tc001.rst @@ -1,4 +1,4 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International +s work is licensed under a Creative Commons Attribution 4.0 International .. License. .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Ericsson AB and others. @@ -13,38 +13,40 @@ Yardstick Test Case Description TC001 |Network Performance | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC001_NW PERF | +|test case id | OPNFV_YARDSTICK_TC001_NETWORK PERFORMANCE | | | | +--------------+--------------------------------------------------------------+ |metric | Number of flows and throughput | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To evaluate the IaaS network performance with regards to | -| | flows and throughput, such as if and how different amounts | -| | of flows matter for the throughput between hosts on | -| | different compute blades. Typically e.g. the performance of | -| | a vSwitch depends on the number of flows running through it. | -| | Also performance of other equipment or entities can depend | -| | on the number of flows or the packet sizes used. | -| | The purpose is also to be able to spot trends. Test results, | -| | graphs ans similar shall be stored for comparison reasons | -| | and product evolution understanding between different OPNFV | -| | versions and/or configurations. | -| | | -+--------------+--------------------------------------------------------------+ -|configuration | file: opnfv_yardstick_tc001.yaml | -| | | -| | Packet size: 60 bytes | -| | Number of ports: 10, 50, 100, 500 and 1000, where each | -| | runs for 20 seconds. The whole sequence is run | -| | twice. The client and server are distributed on different | -| | HW. | -| | For SLA max_ppm is set to 1000. The amount of configured | -| | ports map to between 110 up to 1001000 flows, respectively. | +|test purpose | The purpose of TC001 is to evaluate the IaaS network | +| | performance with regards to flows and throughput, such as if | +| | and how different amounts of flows matter for the throughput | +| | between hosts on different compute blades. Typically e.g. | +| | the performance of a vSwitch depends on the number of flows | +| | running through it. Also performance of other equipment or | +| | entities can depend on the number of flows or the packet | +| | sizes used. | +| | | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | | | | +--------------+--------------------------------------------------------------+ |test tool | pktgen | | | | +| | Linux packet generator is a tool to generate packets at very | +| | high speed in the kernel. pktgen is mainly used to drive and | +| | LAN equipment test network. pktgen supports multi threading. | +| | To generate random MAC address, IP address, port number UDP | +| | packets, pktgen uses multiple CPU processors in the | +| | different PCI bus (PCI, PCIe bus) with Gigabit Ethernet | +| | tested (pktgen performance depends on the CPU processing | +| | speed, memory delay, PCI bus speed hardware parameters), | +| | Transmit data rate can be even larger than 10GBit/s. Visible | +| | can satisfy most card test requirements. | +| | | | | (Pktgen is not always part of a Linux distribution, hence it | | | needs to be installed. It is part of the Yardstick Docker | | | image. | @@ -52,18 +54,47 @@ Yardstick Test Case Description TC001 | | to generate a Linux image with pktgen included.) | | | | +--------------+--------------------------------------------------------------+ -|references | pktgen_ | +|test | This test case uses Pktgen to generate packet flow between | +|description | two hosts for simulating network workloads on the SUT. | | | | -| | ETSI-NFV-TST001 | ++--------------+--------------------------------------------------------------+ +|traffic | An IP table is setup on server to monitor for received | +|profile | packets. | +| | | ++--------------+--------------------------------------------------------------+ +|configuration | file: opnfv_yardstick_tc001.yaml | +| | | +| | Packet size is set to 60 bytes. | +| | Number of ports: 10, 50, 100, 500 and 1000, where each | +| | runs for 20 seconds. The whole sequence is run twice | +| | The client and server are distributed on different hardware. | +| | | +| | For SLA max_ppm is set to 1000. The amount of configured | +| | ports map to between 110 up to 1001000 flows, respectively. | | | | +--------------+--------------------------------------------------------------+ -|applicability | Test can be configured with different packet sizes, amount | -| | of flows and test duration. Default values exist. | +|applicability | Test can be configured with different: | +| | | +| | * packet sizes; | +| | * amount of flows; | +| | * test duration. | +| | | +| | Default values exist. | | | | | | SLA (optional): max_ppm: The number of packets per million | | | packets sent that are acceptable to loose, not received. | | | | +--------------+--------------------------------------------------------------+ +|usability | This test case is used for generating high network | +| | throughput to simulate certain workloads on the SUT. Hence | +| | it should work with other test cases. | +| | | ++--------------+--------------------------------------------------------------+ +|references | pktgen_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | |conditions | with pktgen included in it. | | | | @@ -73,12 +104,29 @@ Yardstick Test Case Description TC001 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The hosts are installed, as server and client. pktgen is | -| | invoked and logs are produced and stored. | +|step 1 | Two host VMs are booted, as server and client. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the server VM by using ssh. | +| | 'pktgen_benchmark' bash script is copyied from Jump Host to | +| | the server VM via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | An IP table is setup on server to monitor for received | +| | packets. | +| | | ++--------------+--------------------------------------------------------------+ +|step 4 | pktgen is invoked to generate packet flow between two server | +| | and client for simulating network workloads on the SUT. | +| | Results are processed and checked against the SLA. Logs are | +| | produced and stored. | | | | | | Result: Logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 5 | Two host VMs are deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Fails only if SLA is not passed, or if there is a test case | | | execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc002.rst b/docs/userguide/opnfv_yardstick_tc002.rst index 193fc531f..c98780fd5 100644 --- a/docs/userguide/opnfv_yardstick_tc002.rst +++ b/docs/userguide/opnfv_yardstick_tc002.rst @@ -8,34 +8,37 @@ Yardstick Test Case Description TC002 ************************************* .. _cirros-image: https://download.cirros-cloud.net +.. _Ping: https://linux.die.net/man/8/ping +-----------------------------------------------------------------------------+ |Network Latency | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC002_NW LATENCY | +|test case id | OPNFV_YARDSTICK_TC002_NETWORK LATENCY | | | | +--------------+--------------------------------------------------------------+ -|metric | RTT, Round Trip Time | +|metric | RTT (Round Trip Time) | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To do a basic verification that network latency is within | -| | acceptable boundaries when packets travel between hosts | -| | located on same or different compute blades. | -| | The purpose is also to be able to spot trends. Test results, | -| | graphs and similar shall be stored for comparison reasons and| -| | product evolution understanding between different OPNFV | -| | versions and/or configurations. | +|test purpose | The purpose of TC002 is to do a basic verification that | +| | network latency is within acceptable boundaries when packets | +| | travel between hosts located on same or different compute | +| | blades. | | | | -+--------------+--------------------------------------------------------------+ -|configuration | file: opnfv_yardstick_tc002.yaml | -| | | -| | Packet size 100 bytes. Total test duration 600 seconds. | -| | One ping each 10 seconds. SLA RTT is set to maximum 10 ms. | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | | | | +--------------+--------------------------------------------------------------+ |test tool | ping | | | | +| | Ping is a computer network administration software utility | +| | used to test the reachability of a host on an Internet | +| | Protocol (IP) network. It measures the round-trip time for | +| | packet sent from the originating host to a destination | +| | computer that are echoed back to the source. | +| | | | | Ping is normally part of any Linux distribution, hence it | | | doesn't need to be installed. It is also part of the | | | Yardstick Docker image. | @@ -43,27 +46,55 @@ Yardstick Test Case Description TC002 | | cirros-image_, it includes ping) | | | | +--------------+--------------------------------------------------------------+ -|references | Ping man page | +|test topology | Ping packets (ICMP protocol's mandatory ECHO_REQUEST | +| | datagram) are sent from host VM to target VM(s) to elicit | +| | ICMP ECHO_RESPONSE. | | | | -| | ETSI-NFV-TST001 | +| | For one host VM there can be multiple target VMs. | +| | Host VM and target VM(s) can be on same or different compute | +| | blades. | +| | | ++--------------+--------------------------------------------------------------+ +|configuration | file: opnfv_yardstick_tc002.yaml | +| | | +| | Packet size 100 bytes. Test duration 60 seconds. | +| | One ping each 10 seconds. Test is iterated two times. | +| | SLA RTT is set to maximum 10 ms. | | | | +--------------+--------------------------------------------------------------+ -|applicability | Test case can be configured with different packet sizes, | -| | burst sizes, ping intervals and test duration. | +|applicability | This test case can be configured with different: | +| | | +| | * packet sizes; | +| | * burst sizes; | +| | * ping intervals; | +| | * test durations; | +| | * test iterations. | +| | | +| | Default values exist. | +| | | | | SLA is optional. The SLA in this test case serves as an | -| | example. Considerably lower RTT is expected, and | -| | also normal to achieve in balanced L2 environments. However, | -| | to cover most configurations, both bare metal and fully | -| | virtualized ones, this value should be possible to achieve | -| | and acceptable for black box testing. Many real time | +| | example. Considerably lower RTT is expected, and also normal | +| | to achieve in balanced L2 environments. However, to cover | +| | most configurations, both bare metal and fully virtualized | +| | ones, this value should be possible to achieve and | +| | acceptable for black box testing. Many real time | | | applications start to suffer badly if the RTT time is higher | | | than this. Some may suffer bad also close to this RTT, while | | | others may not suffer at all. It is a compromise that may | | | have to be tuned for different configuration purposes. | | | | +--------------+--------------------------------------------------------------+ -|pre-test | The test case image needs to be installed into Glance | -|conditions | with ping included in it. | +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | +| | | ++--------------+--------------------------------------------------------------+ +|references | Ping_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ +|pre-test | The test case image (cirros-image) needs to be installed | +|conditions | into Glance with ping included in it. | | | | | | No POD specific requirements have been identified. | | | | @@ -71,12 +102,24 @@ Yardstick Test Case Description TC002 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The hosts are installed, as server and client. Ping is | -| | invoked and logs are produced and stored. | +|step 1 | Two host VMs are booted, as server and client. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the server VM by using ssh. | +| | 'ping_benchmark' bash script is copyied from Jump Host to | +| | the server VM via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | Ping is invoked. Ping packets are sent from server VM to | +| | client VM. RTT results are calculated and checked against | +| | the SLA. Logs are produced and stored. | | | | | | Result: Logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 4 | Two host VMs are deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Test should not PASS if any RTT is above the optional SLA | | | value, or if there is a test case execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc004.rst b/docs/userguide/opnfv_yardstick_tc004.rst index 301286126..3554b3826 100644 --- a/docs/userguide/opnfv_yardstick_tc004.rst +++ b/docs/userguide/opnfv_yardstick_tc004.rst @@ -13,39 +13,52 @@ Yardstick Test Case Description TC004 |Cache Utilization | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC004_Cache Utilization | +|test case id | OPNFV_YARDSTICK_TC004_CACHE Utilization | | | | +--------------+--------------------------------------------------------------+ -|metric | Cache Utilization | +|metric | cache hit, cache miss, hit/miss ratio, buffer size and page | +| | cache size | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To evaluate the IaaS compute capability with regards to | -| | cache utilization.This test case should be run in parallel | -| | to other Yardstick test cases and not run as a stand-alone | -| | test case. | -| | Measure the cache usage statistics including cache hit, | -| | cache miss, hit ratio, page cache size and page cache size. | -| | Both average and maximun values are obtained. | -| | The purpose is also to be able to spot trends. | +|test purpose | The purpose of TC004 is to evaluate the IaaS compute | +| | capability with regards to cache utilization.This test case | +| | should be run in parallel with other Yardstick test cases | +| | and not run as a stand-alone test case. | +| | | +| | This test case measures cache usage statistics, including | +| | cache hit, cache miss, hit ratio, buffer cache size and page | +| | cache size, with some wokloads runing on the infrastructure. | +| | Both average and maximun values are collected. | +| | | +| | The purpose is also to be able to spot the trends. | | | Test results, graphs and similar shall be stored for | | | comparison reasons and product evolution understanding | | | between different OPNFV versions and/or configurations. | | | | +--------------+--------------------------------------------------------------+ -|configuration | File: cachestat.yaml (in the 'samples' directory) | +|test tool | cachestat | | | | -| | * interval: 1 - repeat, pausing every 1 seconds in-between. | +| | cachestat is a tool using Linux ftrace capabilities for | +| | showing Linux page cache hit/miss statistics. | | | | -+--------------+--------------------------------------------------------------+ -|test tool | cachestat | +| | (cachestat is not always part of a Linux distribution, hence | +| | it needs to be installed. As an example see the | +| | /yardstick/tools/ directory for how to generate a Linux | +| | image with cachestat included.) | | | | -| | cachestat is not always part of a Linux distribution, hence | -| | it needs to be installed. | ++--------------+--------------------------------------------------------------+ +|test | cachestat test is invoked in a host VM on a compute blade, | +|description | cachestat test requires some other test cases running in the | +| | host to stimulate workload. | | | | +--------------+--------------------------------------------------------------+ -|references | cachestat_ | +|configuration | File: cachestat.yaml (in the 'samples' directory) | | | | -| | ETSI-NFV-TST001 | +| | Interval is set 1. Test repeat, pausing every 1 seconds | +| | in-between. | +| | Test durarion is set to 60 seconds. | +| | | +| | SLA is not available in this test case. | | | | +--------------+--------------------------------------------------------------+ |applicability | Test can be configured with different: | @@ -53,8 +66,16 @@ Yardstick Test Case Description TC004 | | * interval; | | | * runner Duration. | | | | -| | There are default values for each above-mentioned option. | -| | Run in background with other test cases. | +| | Default values exist. | +| | | ++--------------+--------------------------------------------------------------+ +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | +| | | ++--------------+--------------------------------------------------------------+ +|references | cachestat_ | +| | | +| | ETSI-NFV-TST001 | | | | +--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | @@ -66,12 +87,24 @@ Yardstick Test Case Description TC004 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The host is installed as client. The related TC, or TCs, is | -| | invoked and cachestat logs are produced and stored. | +|step 1 | A host VM with cachestat installed is booted. | | | | -| | Result: logs are stored. | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the host VM by using ssh. | +| | 'cache_stat' bash script is copyied from Jump Host to | +| | the server VM via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | 'cache_stat' script is invoked. Raw cache usage statistics | +| | are collected and filtrated. Average and maximum values are | +| | calculated and recorded. Logs are produced and stored. | +| | | +| | Result: Logs are stored. | +| | | ++--------------+--------------------------------------------------------------+ +|step 4 | The host VM is deleted. | | | | +--------------+--------------------------------------------------------------+ -|test verdict | None. Cache utilization results are fetched and stored. | +|test verdict | None. Cache utilization results are collected and stored. | | | | +--------------+--------------------------------------------------------------+ diff --git a/docs/userguide/opnfv_yardstick_tc005.rst b/docs/userguide/opnfv_yardstick_tc005.rst index a181aa9f7..1c2d71d81 100644 --- a/docs/userguide/opnfv_yardstick_tc005.rst +++ b/docs/userguide/opnfv_yardstick_tc005.rst @@ -1,4 +1,4 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International +. This work is licensed under a Creative Commons Attribution 4.0 International .. License. .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Huawei Technologies Co.,Ltd and others. @@ -7,53 +7,88 @@ Yardstick Test Case Description TC005 ************************************* -.. _fio: http://www.bluestop.org/fio/HOWTO.txt +.. _fio: http://bluestop.org/files/fio/HOWTO.txt +-----------------------------------------------------------------------------+ |Storage Performance | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC005_Storage Performance | +|test case id | OPNFV_YARDSTICK_TC005_STORAGE PERFORMANCE | | | | +--------------+--------------------------------------------------------------+ -|metric | IOPS, throughput and latency | +|metric | IOPS (Average IOs performed per second), | +| | Throughput (Average disk read/write bandwidth rate), | +| | Latency (Average disk read/write latency) | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To evaluate the IaaS storage performance with regards to | -| | IOPS, throughput and latency. | -| | The purpose is also to be able to spot trends. Test results, | -| | graphs and similar shall be stored for comparison reasons | -| | and product evolution understanding between different OPNFV | -| | versions and/or configurations. | +|test purpose | The purpose of TC005 is to evaluate the IaaS storage | +| | performance with regards to IOPS, throughput and latency. | | | | -+--------------+--------------------------------------------------------------+ -|configuration | file: opnfv_yardstick_tc005.yaml | -| | | -| | IO types: read, write, randwrite, randread, rw | -| | IO block size: 4KB, 64KB, 1024KB, where each | -| | runs for 30 seconds(10 for ramp time, 20 for runtime). | -| | | -| | For SLA minimum read/write iops is set to 100, minimum | -| | read/write throughput is set to 400 KB/s, and maximum | -| | read/write latency is set to 20000 usec. | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | | | | +--------------+--------------------------------------------------------------+ |test tool | fio | | | | +| | fio is an I/O tool meant to be used both for benchmark and | +| | stress/hardware verification. It has support for 19 | +| | different types of I/O engines (sync, mmap, libaio, | +| | posixaio, SG v3, splice, null, network, syslet, guasi, | +| | solarisaio, and more), I/O priorities (for newer Linux | +| | kernels), rate I/O, forked or threaded jobs, and much more. | +| | | | | (fio is not always part of a Linux distribution, hence it | | | needs to be installed. As an example see the | | | /yardstick/tools/ directory for how to generate a Linux | | | image with fio included.) | | | | +--------------+--------------------------------------------------------------+ -|references | fio_ | +|test | fio test is invoked in a host VM on a compute blade, a job | +|description | file as well as parameters are passed to fio and fio will | +| | start doing what the job file tells it to do. | | | | -| | ETSI-NFV-TST001 | ++--------------+--------------------------------------------------------------+ +|configuration | file: opnfv_yardstick_tc005.yaml | +| | | +| | IO types is set to read, write, randwrite, randread, rw. | +| | IO block size is set to 4KB, 64KB, 1024KB. | +| | fio is run for each IO type and IO block size scheme, | +| | each iteration runs for 30 seconds (10 for ramp time, 20 for | +| | runtime). | +| | | +| | For SLA, minimum read/write iops is set to 100, | +| | minimum read/write throughput is set to 400 KB/s, | +| | and maximum read/write latency is set to 20000 usec. | | | | +--------------+--------------------------------------------------------------+ -|applicability | Test can be configured with different read/write types, IO | -| | block size, IO depth, ramp time (runtime required for stable | -| | results) and test duration. Default values exist. | +|applicability | This test case can be configured with different: | +| | | +| | * IO types; | +| | * IO block size; | +| | * IO depth; | +| | * ramp time; | +| | * test duration. | +| | | +| | Default values exist. | +| | | +| | SLA is optional. The SLA in this test case serves as an | +| | example. Considerably higher throughput and lower latency | +| | are expected. However, to cover most configurations, both | +| | baremetal and fully virtualized ones, this value should be | +| | possible to achieve and acceptable for black box testing. | +| | Many heavy IO applications start to suffer badly if the | +| | read/write bandwidths are lower than this. | +| | | ++--------------+--------------------------------------------------------------+ +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | +| | | ++--------------+--------------------------------------------------------------+ +|references | fio_ | +| | | +| | ETSI-NFV-TST001 | | | | +--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | @@ -65,12 +100,25 @@ Yardstick Test Case Description TC005 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The host is installed and fio is invoked and logs are | -| | produced and stored. | +|step 1 | A host VM with fio installed is booted. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the host VM by using ssh. | +| | 'fio_benchmark' bash script is copyied from Jump Host to | +| | the host VM via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | 'fio_benchmark' script is invoked. Simulated IO operations | +| | are started. IOPS, disk read/write bandwidth and latency are | +| | recorded and checked against the SLA. Logs are produced and | +| | stored. | | | | | | Result: Logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 4 | The host VM is deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Fails only if SLA is not passed, or if there is a test case | | | execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc010.rst b/docs/userguide/opnfv_yardstick_tc010.rst index ab793de76..202307de6 100644 --- a/docs/userguide/opnfv_yardstick_tc010.rst +++ b/docs/userguide/opnfv_yardstick_tc010.rst @@ -7,21 +7,71 @@ Yardstick Test Case Description TC010 ************************************* -.. _man-pages: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html +.. _lat_mem_rd: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html +-----------------------------------------------------------------------------+ |Memory Latency | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC010_Memory Latency | +|test case id | OPNFV_YARDSTICK_TC010_MEMORY LATENCY | | | | +--------------+--------------------------------------------------------------+ -|metric | Latency in nanoseconds | +|metric | Memory read latency (nanoseconds) | | | | +--------------+--------------------------------------------------------------+ -|test purpose | Measure the memory read latency for varying memory sizes and | -| | strides. Whole memory hierarchy is measured including all | -| | levels of cache. | +|test purpose | The purpose of TC010 is to evaluate the IaaS compute | +| | performance with regards to memory read latency. | +| | It measures the memory read latency for varying memory sizes | +| | and strides. Whole memory hierarchy is measured. | +| | | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | Lmbench | +| | | +| | Lmbench is a suite of operating system microbenchmarks. This | +| | test uses lat_mem_rd tool from that suite including: | +| | * Context switching | +| | * Networking: connection establishment, pipe, TCP, UDP, and | +| | RPC hot potato | +| | * File system creates and deletes | +| | * Process creation | +| | * Signal handling | +| | * System call overhead | +| | * Memory read latency | +| | | +| | (LMbench is not always part of a Linux distribution, hence | +| | it needs to be installed. As an example see the | +| | /yardstick/tools/ directory for how to generate a Linux | +| | image with LMbench included.) | +| | | ++--------------+--------------------------------------------------------------+ +|test | LMbench lat_mem_rd benchmark measures memory read latency | +|description | for varying memory sizes and strides. | +| | | +| | The benchmark runs as two nested loops. The outer loop is | +| | the stride size. The inner loop is the array size. For each | +| | array size, the benchmark creates a ring of pointers that | +| | point backward one stride.Traversing the array is done by: | +| | | +| | p = (char **)*p; | +| | | +| | in a for loop (the over head of the for loop is not | +| | significant; the loop is an unrolled loop 100 loads long). | +| | The size of the array varies from 512 bytes to (typically) | +| | eight megabytes. For the small sizes, the cache will have an | +| | effect, and the loads will be much faster. This becomes much | +| | more apparent when the data is plotted. | +| | | +| | Only data accesses are measured; the instruction cache is | +| | not measured. | +| | | +| | The results are reported in nanoseconds per load and have | +| | been verified accurate to within a few nanoseconds on an SGI | +| | Indy. | | | | +--------------+--------------------------------------------------------------+ |configuration | File: opnfv_yardstick_tc010.yaml | @@ -33,20 +83,13 @@ Yardstick Test Case Description TC010 | | * Interval: 1 - there is 1 second delay between each | | | iteration. | | | | -+--------------+--------------------------------------------------------------+ -|test tool | Lmbench | -| | | -| | Lmbench is a suite of operating system microbenchmarks. This | -| | test uses lat_mem_rd tool from that suite. | -| | Lmbench is not always part of a Linux distribution, hence it | -| | needs to be installed in the test image | -| | | -+--------------+--------------------------------------------------------------+ -|references | man-pages_ | -| | | -| | McVoy, Larry W.,and Carl Staelin. "lmbench: Portable Tools | -| | for Performance Analysis." USENIX annual technical | -| | conference 1996. | +| | SLA is optional. The SLA in this test case serves as an | +| | example. Considerably lower read latency is expected. | +| | However, to cover most configurations, both baremetal and | +| | fully virtualized ones, this value should be possible to | +| | achieve and acceptable for black box testing. | +| | Many heavy IO applications start to suffer badly if the | +| | read latency is higher than this. | | | | +--------------+--------------------------------------------------------------+ |applicability | Test can be configured with different: | @@ -55,12 +98,21 @@ Yardstick Test Case Description TC010 | | * stop_size; | | | * iterations and intervals. | | | | -| | There are default values for each above-mentioned option. | +| | Default values exist. | | | | | | SLA (optional) : max_latency: The maximum memory latency | | | that is accepted. | | | | +--------------+--------------------------------------------------------------+ +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | +| | | ++--------------+--------------------------------------------------------------+ +|references | LMbench lat_mem_rd_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | |conditions | with Lmbench included in the image. | | | | @@ -70,12 +122,32 @@ Yardstick Test Case Description TC010 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The host is installed as client. Lmbench's lat_mem_rd tool | +|step 1 | The host is installed as client. LMbench's lat_mem_rd tool | | | is invoked and logs are produced and stored. | | | | | | Result: logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 1 | A host VM with LMbench installed is booted. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the host VM by using ssh. | +| | 'lmbench_latency_benchmark' bash script is copyied from Jump | +| | Host to the host VM via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | 'lmbench_latency_benchmark' script is invoked. LMbench's | +| | lat_mem_rd benchmark starts to measures memory read latency | +| | for varying memory sizes and strides. Memory read latency | +| | are recorded and checked against the SLA. Logs are produced | +| | and stored. | +| | | +| | Result: Logs are stored. | +| | | ++--------------+--------------------------------------------------------------+ +|step 4 | The host VM is deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Test fails if the measured memory latency is above the SLA | | | value or if there is a test case execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc011.rst b/docs/userguide/opnfv_yardstick_tc011.rst index cf2fd5055..48bdef497 100644 --- a/docs/userguide/opnfv_yardstick_tc011.rst +++ b/docs/userguide/opnfv_yardstick_tc011.rst @@ -13,28 +13,22 @@ Yardstick Test Case Description TC011 |Packet delay variation between VMs | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC011_Packet delay variation between VMs | +|test case id | OPNFV_YARDSTICK_TC011_PACKET DELAY VARIATION BETWEEN VMs | | | | +--------------+--------------------------------------------------------------+ |metric | jitter: packet delay variation (ms) | | | | +--------------+--------------------------------------------------------------+ -|test purpose | Measure the packet delay variation sending the packets from | -| | one VM to the other. | +|test purpose | The purpose of TC011 is to evaluate the IaaS network | +| | performance with regards to network jitter (packet delay | +| | variation). | +| | It measures the packet delay variation sending the packets | +| | from one VM to the other. | | | | -+--------------+--------------------------------------------------------------+ -|configuration | File: opnfv_yardstick_tc011.yaml | -| | | -| | * options: | -| | protocol: udp # The protocol used by iperf3 tools | -| | bandwidth: 20m # It will send the given number of packets | -| | without pausing | -| | * runner: | -| | duration: 30 # Total test duration 30 seconds. | -| | | -| | * SLA (optional): | -| | jitter: 10 (ms) # The maximum amount of jitter that is | -| | accepted. | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | | | | +--------------+--------------------------------------------------------------+ |test tool | iperf3 | @@ -46,14 +40,34 @@ Yardstick Test Case Description TC011 | | | | | (iperf3 is not always part of a Linux distribution, hence it | | | needs to be installed. It is part of the Yardstick Docker | -| | image. | -| | As an example see the /yardstick/tools/ directory for how | -| | to generate a Linux image with pktgen included.) | +| | image. As an example see the /yardstick/tools/ directory for | +| | how to generate a Linux image with pktgen included.) | | | | +--------------+--------------------------------------------------------------+ -|references | iperf3_ | +|test | iperf3 test is invoked between a host VM and a target VM. | +|description | | +| | Jitter calculations are continuously computed by the server, | +| | as specified by RTP in RFC 1889. The client records a 64 bit | +| | second/microsecond timestamp in the packet. The server | +| | computes the relative transit time as (server's receive time | +| | - client's send time). The client's and server's clocks do | +| | not need to be synchronized; any difference is subtracted | +| | outin the jitter calculation. Jitter is the smoothed mean of | +| | differences between consecutive transit times. | | | | -| | ETSI-NFV-TST001 | ++--------------+--------------------------------------------------------------+ +|configuration | File: opnfv_yardstick_tc011.yaml | +| | | +| | * options: | +| | protocol: udp # The protocol used by iperf3 tools | +| | bandwidth: 20m # It will send the given number of packets | +| | without pausing | +| | * runner: | +| | duration: 30 # Total test duration 30 seconds. | +| | | +| | * SLA (optional): | +| | jitter: 10 (ms) # The maximum amount of jitter that is | +| | accepted. | | | | +--------------+--------------------------------------------------------------+ |applicability | Test can be configured with different: | @@ -67,6 +81,15 @@ Yardstick Test Case Description TC011 | | serves as an example. | | | | +--------------+--------------------------------------------------------------+ +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | +| | | ++--------------+--------------------------------------------------------------+ +|references | iperf3_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | |conditions | with iperf3 included in the image. | | | | @@ -76,12 +99,24 @@ Yardstick Test Case Description TC011 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The hosts are installed, as server and client. iperf3 is | -| | invoked and logs are produced and stored. | +|step 1 | Two host VMs with iperf3 installed are booted, as server and | +| | client. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the host VM by using ssh. | +| | A iperf3 server is started on the server VM via the ssh | +| | tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | iperf3 benchmark is invoked. Jitter is calculated and check | +| | against the SLA. Logs are produced and stored. | | | | | | Result: Logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 4 | The host VMs are deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Test should not PASS if any jitter is above the optional SLA | | | value, or if there is a test case execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc012.rst b/docs/userguide/opnfv_yardstick_tc012.rst index ffce06eb9..b56e829f5 100644 --- a/docs/userguide/opnfv_yardstick_tc012.rst +++ b/docs/userguide/opnfv_yardstick_tc012.rst @@ -7,29 +7,60 @@ Yardstick Test Case Description TC012 ************************************* -.. _man-pages: http://manpages.ubuntu.com/manpages/trusty/bw_mem.8.html +.. _bw_mem: http://manpages.ubuntu.com/manpages/trusty/bw_mem.8.html +-----------------------------------------------------------------------------+ |Memory Bandwidth | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC012_Memory Bandwidth | +|test case id | OPNFV_YARDSTICK_TC012_MEMORY BANDWIDTH | | | | +--------------+--------------------------------------------------------------+ -|metric | Megabyte per second (MBps) | +|metric | Memory read/write bandwidth (MBps) | | | | +--------------+--------------------------------------------------------------+ -|test purpose | Measure the rate at which data can be read from and written | -| | to the memory (this includes all levels of memory). | +|test purpose | The purpose of TC012 is to evaluate the IaaS compute | +| | performance with regards to memory throughput. | +| | It measures the rate at which data can be read from and | +| | written to the memory (this includes all levels of memory). | +| | | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | LMbench | +| | | +| | LMbench is a suite of operating system microbenchmarks. | +| | This test uses bw_mem tool from that suite including: | +| | * Cached file read | +| | * Memory copy (bcopy) | +| | * Memory read | +| | * Memory write | +| | * Pipe | +| | * TCP | +| | | +| | (LMbench is not always part of a Linux distribution, hence | +| | it needs to be installed. As an example see the | +| | /yardstick/tools/ directory for how to generate a Linux | +| | image with LMbench included.) | +| | | ++--------------+--------------------------------------------------------------+ +|test | LMbench bw_mem benchmark allocates twice the specified | +|description | amount of memory, zeros it, and then times the copying of | +| | the first half to the second half. The benchmark is invoked | +| | in a host VM on a compute blade. Results are reported in | +| | megabytes moved per second. | | | | +--------------+--------------------------------------------------------------+ |configuration | File: opnfv_yardstick_tc012.yaml | | | | | | * SLA (optional): 15000 (MBps) min_bw: The minimum amount of | | | memory bandwidth that is accepted. | -| | * Size: 10 240 kB - test allocates twice that size (20 480kB)| -| | zeros it and then measures the time it takes to copy from | -| | one side to another. | +| | * Size: 10 240 kB - test allocates twice that size | +| | (20 480kB) zeros it and then measures the time it takes to | +| | copy from one side to another. | | | * Benchmark: rdwr - measures the time to read data into | | | memory and then write data to the same location. | | | * Warmup: 0 - the number of iterations to perform before | @@ -38,20 +69,13 @@ Yardstick Test Case Description TC012 | | * Interval: 1 - there is 1 second delay between each | | | iteration. | | | | -+--------------+--------------------------------------------------------------+ -|test tool | Lmbench | -| | | -| | Lmbench is a suite of operating system microbenchmarks. This | -| | test uses bw_mem tool from that suite. | -| | Lmbench is not always part of a Linux distribution, hence it | -| | needs to be installed in the test image. | -| | | -+--------------+--------------------------------------------------------------+ -|references | man-pages_ | -| | | -| | McVoy, Larry W., and Carl Staelin. "lmbench: Portable Tools | -| | for Performance Analysis." USENIX annual technical | -| | conference. 1996. | +| | SLA is optional. The SLA in this test case serves as an | +| | example. Considerably higher bandwidth is expected. | +| | However, to cover most configurations, both baremetal and | +| | fully virtualized ones, this value should be possible to | +| | achieve and acceptable for black box testing. | +| | Many heavy IO applications start to suffer badly if the | +| | read/write bandwidths are lower than this. | | | | +--------------+--------------------------------------------------------------+ |applicability | Test can be configured with different: | @@ -62,7 +86,19 @@ Yardstick Test Case Description TC012 | | * number of warmup iterations; | | | * iterations and intervals. | | | | -| | There are default values for each above-mentioned option. | +| | Default values exist. | +| | | +| | SLA (optional) : min_bandwidth: The minimun memory bandwidth | +| | that is accepted. | +| | | ++--------------+--------------------------------------------------------------+ +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | +| | | ++--------------+--------------------------------------------------------------+ +|references | LMbench bw_mem_ | +| | | +| | ETSI-NFV-TST001 | | | | +--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | @@ -74,10 +110,23 @@ Yardstick Test Case Description TC012 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The host is installed as client. Lmbench's bw_mem tool is | -| | invoked and logs are produced and stored. | +|step 1 | A host VM with LMbench installed is booted. | | | | -| | Result: logs are stored. | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the host VM by using ssh. | +| | "lmbench_bandwidth_benchmark" bash script is copied from | +| | Jump Host to the host VM via ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | 'lmbench_bandwidth_benchmark' script is invoked. LMbench's | +| | bw_mem benchmark starts to measures memory read/write | +| | bandwidth. Memory read/write bandwidth results are recorded | +| | and checked against the SLA. Logs are produced and stored. | +| | | +| | Result: Logs are stored. | +| | | ++--------------+--------------------------------------------------------------+ +|step 4 | The host VM is deleted. | | | | +--------------+--------------------------------------------------------------+ |test verdict | Test fails if the measured memory bandwidth is below the SLA | diff --git a/docs/userguide/opnfv_yardstick_tc014.rst b/docs/userguide/opnfv_yardstick_tc014.rst index 27d390ac6..1b0d7831a 100644 --- a/docs/userguide/opnfv_yardstick_tc014.rst +++ b/docs/userguide/opnfv_yardstick_tc014.rst @@ -13,18 +13,51 @@ Yardstick Test Case Description TC014 |Processing speed | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC014_Processing speed | +|test case id | OPNFV_YARDSTICK_TC014_PROCESSING SPEED | | | | +--------------+--------------------------------------------------------------+ -|metric | score of single cpu running, score of parallel running | +|metric | score of single cpu running, | +| | score of parallel running | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To evaluate the IaaS processing speed with regards to score | -| | of single cpu running and parallel running | -| | The purpose is also to be able to spot trends. Test results, | -| | graphs and similar shall be stored for comparison reasons | -| | and product evolution understanding between different OPNFV | -| | versions and/or configurations. | +|test purpose | The purpose of TC014 is to evaluate the IaaS compute | +| | performance with regards to CPU processing speed. | +| | It measures score of single cpu running and parallel | +| | running. | +| | | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | UnixBench | +| | | +| | Unixbench is the most used CPU benchmarking software tool. | +| | It can measure the performance of bash scripts, CPUs in | +| | multithreading and single threading. It can also measure the | +| | performance for parallel taks. Also, specific disk IO for | +| | small and large files are performed. You can use it to | +| | measure either linux dedicated servers and linux vps | +| | servers, running CentOS, Debian, Ubuntu, Fedora and other | +| | distros. | +| | | +| | (UnixBench is not always part of a Linux distribution, hence | +| | it needs to be installed. As an example see the | +| | /yardstick/tools/ directory for how to generate a Linux | +| | image with UnixBench included.) | +| | | ++--------------+--------------------------------------------------------------+ +|test | The UnixBench runs system benchmarks in a host VM on a | +|description | compute blade, getting information on the CPUs in the | +| | system. If the system has more than one CPU, the tests will | +| | be run twice -- once with a single copy of each test running | +| | at once, and once with N copies, where N is the number of | +| | CPUs. | +| | | +| | UnixBench will processs a set of results from a single test | +| | by averaging the individal pass results into a single final | +| | value. | | | | +--------------+--------------------------------------------------------------+ |configuration | file: opnfv_yardstick_tc014.yaml | @@ -33,15 +66,23 @@ Yardstick Test Case Description TC014 | | test_type: dhry2reg, whetstone and so on | | | | | | For SLA with single_score and parallel_score, both can be | -| | set by user, default is NA | +| | set by user, default is NA. | | | | +--------------+--------------------------------------------------------------+ -|test tool | unixbench | +|applicability | Test can be configured with different: | | | | -| | (unixbench is not always part of a Linux distribution, hence | -| | it needs to be installed. As an example see the | -| | /yardstick/tools/ directory for how to generate a Linux | -| | image with unixbench included.) | +| | * test types; | +| | * dhry2reg; | +| | * whetstone. | +| | | +| | Default values exist. | +| | | +| | SLA (optional) : min_score: The minimun UnixBench score that | +| | is accepted. | +| | | ++--------------+--------------------------------------------------------------+ +|usability | This test case is one of Yardstick's generic test. Thus it | +| | is runnable on most of the scenarios. | | | | +--------------+--------------------------------------------------------------+ |references | unixbench_ | @@ -49,10 +90,6 @@ Yardstick Test Case Description TC014 | | ETSI-NFV-TST001 | | | | +--------------+--------------------------------------------------------------+ -|applicability | Test can be configured with different test types, dhry2reg, | -| | whetstone and so on. | -| | | -+--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | |conditions | with unixbench included in it. | | | | @@ -62,12 +99,27 @@ Yardstick Test Case Description TC014 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The hosts are installed, as a client. unixbench is | -| | invoked and logs are produced and stored. | +|step 1 | A host VM with UnixBench installed is booted. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the host VM by using ssh. | +| | "unixbench_benchmark" bash script is copied from Jump Host | +| | to the host VM via ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | UnixBench is invoked. All the tests are executed using the | +| | "Run" script in the top-level of UnixBench directory. | +| | The "Run" script will run a standard "index" test, and save | +| | the report in the "results" directory. Then the report is | +| | processed by "unixbench_benchmark" and checked againsted the | +| | SLA. | | | | | | Result: Logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 4 | The host VM is deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Fails only if SLA is not passed, or if there is a test case | | | execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc037.rst b/docs/userguide/opnfv_yardstick_tc037.rst index 3ed1fa529..5a6e1eaae 100644 --- a/docs/userguide/opnfv_yardstick_tc037.rst +++ b/docs/userguide/opnfv_yardstick_tc037.rst @@ -7,84 +7,128 @@ Yardstick Test Case Description TC037 ************************************* -.. _cirros: https://download.cirros-cloud.net +.. _cirros-image: https://download.cirros-cloud.net +.. _Ping: https://linux.die.net/man/8/ping .. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt +.. _mpstat: http://www.linuxcommand.org/man_pages/mpstat1.html +-----------------------------------------------------------------------------+ |Latency, CPU Load, Throughput, Packet Loss | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC037_Latency,CPU Load,Throughput,Packet Loss| +|test case id | OPNFV_YARDSTICK_TC037_LATENCY,CPU LOAD,THROUGHPUT, | +| | PACKET LOSS | | | | +--------------+--------------------------------------------------------------+ -|metric | Number of flows, latency, throughput, CPU load, packet loss | +|metric | Number of flows, latency, throughput, packet loss | +| | CPU utilization percentage, CPU interrupt per second | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To evaluate the IaaS network performance with regards to | -| | flows and throughput, such as if and how different amounts | -| | of flows matter for the throughput between hosts on different| -| | compute blades. Typically e.g. the performance of a vSwitch | -| | depends on the number of flows running through it. Also | -| | performance of other equipment or entities can depend | -| | on the number of flows or the packet sizes used. | -| | The purpose is also to be able to spot trends. Test results, | -| | graphs ans similar shall be stored for comparison reasons and| -| | product evolution understanding between different OPNFV | -| | versions and/or configurations. | +|test purpose | The purpose of TC037 is to evaluate the IaaS compute | +| | capacity and network performance with regards to CPU | +| | utilization, packet flows and network throughput, such as if | +| | and how different amounts of flows matter for the throughput | +| | between hosts on different compute blades, and the CPU load | +| | variation. | +| | | +| | Typically e.g. the performance of a vSwitch depends on the | +| | number of flows running through it. Also performance of | +| | other equipment or entities can depend on the number of | +| | flows or the packet sizes used | +| | | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | Ping, Pktgen, mpstat | +| | | +| | Ping is a computer network administration software utility | +| | used to test the reachability of a host on an Internet | +| | Protocol (IP) network. It measures the round-trip time for | +| | packet sent from the originating host to a destination | +| | computer that are echoed back to the source. | +| | | +| | Linux packet generator is a tool to generate packets at very | +| | high speed in the kernel. pktgen is mainly used to drive and | +| | LAN equipment test network. pktgen supports multi threading. | +| | To generate random MAC address, IP address, port number UDP | +| | packets, pktgen uses multiple CPU processors in the | +| | different PCI bus (PCI, PCIe bus) with Gigabit Ethernet | +| | tested (pktgen performance depends on the CPU processing | +| | speed, memory delay, PCI bus speed hardware parameters), | +| | Transmit data rate can be even larger than 10GBit/s. Visible | +| | can satisfy most card test requirements. | +| | | +| | The mpstat command writes to standard output activities for | +| | each available processor, processor 0 being the first one. | +| | Global average activities among all processors are also | +| | reported. The mpstat command can be used both on SMP and UP | +| | machines, but in the latter, only global average activities | +| | will be printed. | +| | | +| | (Ping is normally part of any Linux distribution, hence it | +| | doesn't need to be installed. It is also part of the | +| | Yardstick Docker image. | +| | For example also a Cirros image can be downloaded from | +| | cirros-image_, it includes ping. | +| | | +| | Pktgen and mpstat are not always part of a Linux | +| | distribution, hence it needs to be installed. It is part of | +| | the Yardstick Docker image. | +| | As an example see the /yardstick/tools/ directory for how | +| | to generate a Linux image with pktgen and mpstat included.) | +| | | ++--------------+--------------------------------------------------------------+ +|test | This test case uses Pktgen to generate packet flow between | +|description | two hosts for simulating network workloads on the SUT. | +| | Ping packets (ICMP protocol's mandatory ECHO_REQUEST | +| | datagram) are sent from a host VM to the target VM(s) to | +| | elicit ICMP ECHO_RESPONSE, meanwhile CPU activities are | +| | monitored by mpstat. | | | | +--------------+--------------------------------------------------------------+ |configuration | file: opnfv_yardstick_tc037.yaml | | | | -| | Packet size: 64 bytes | +| | Packet size is set to 64 bytes. | | | Number of ports: 1, 10, 50, 100, 300, 500, 750 and 1000. | | | The amount configured ports map from 2 up to 1001000 flows, | | | respectively. Each port amount is run two times, for 20 | | | seconds each. Then the next port_amount is run, and so on. | | | During the test CPU load on both client and server, and the | | | network latency between the client and server are measured. | -| | The client and server are distributed on different HW. | +| | The client and server are distributed on different hardware. | +| | mpstat monitoring interval is set to 1 second. | +| | ping packet size is set to 100 bytes. | | | For SLA max_ppm is set to 1000. | | | | +--------------+--------------------------------------------------------------+ -|test tool | pktgen | +|applicability | Test can be configured with different: | | | | -| | (Pktgen is not always part of a Linux distribution, hence it | -| | needs to be installed. It is part of the Yardstick Glance | -| | image. | -| | As an example see the /yardstick/tools/ directory for how | -| | to generate a Linux image with pktgen included.) | -| | | -| | ping | -| | | -| | Ping is normally part of any Linux distribution, hence it | -| | doesn't need to be installed. It is also part of the | -| | Yardstick Glance image. | -| | (For example also a cirros_ image can be downloaded, it | -| | includes ping) | +| | * pktgen packet sizes; | +| | * amount of flows; | +| | * test duration; | +| | * ping packet size; | +| | * mpstat monitor interval. | | | | -| | mpstat | +| | Default values exist. | | | | -| | (Mpstat is not always part of a Linux distribution, hence it | -| | needs to be installed. It is part of the Yardstick Glance | -| | image. | +| | SLA (optional): max_ppm: The number of packets per million | +| | packets sent that are acceptable to loose, not received. | | | | +--------------+--------------------------------------------------------------+ -|references | Ping and Mpstat man pages | +|references | Ping_ | +| | | +| | mpstat_ | | | | | | pktgen_ | | | | | | ETSI-NFV-TST001 | | | | +--------------+--------------------------------------------------------------+ -|applicability | Test can be configured with different packet sizes, amount | -| | of flows and test duration. Default values exist. | -| | | -| | SLA (optional): max_ppm: The number of packets per million | -| | packets sent that are acceptable to loose, not received. | -| | | -+--------------+--------------------------------------------------------------+ |pre-test | The test case image needs to be installed into Glance | -|conditions | with pktgen included in it. | +|conditions | with pktgen, mpstat included in it. | | | | | | No POD specific requirements have been identified. | | | | @@ -92,12 +136,31 @@ Yardstick Test Case Description TC037 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The hosts are installed, as server and client. pktgen is | -| | invoked and logs are produced and stored. | +|step 1 | Two host VMs are booted, as server and client. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Yardstick is connected with the server VM by using ssh. | +| | 'pktgen_benchmark', "ping_benchmark" bash script are copyied | +| | from Jump Host to the server VM via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | An IP table is setup on server to monitor for received | +| | packets. | +| | | ++--------------+--------------------------------------------------------------+ +|step 4 | pktgen is invoked to generate packet flow between two server | +| | and client for simulating network workloads on the SUT. Ping | +| | is invoked. Ping packets are sent from server VM to client | +| | VM. mpstat is invoked, recording activities for each | +| | available processor. Results are processed and checked | +| | against the SLA. Logs are produced and stored. | | | | | | Result: Logs are stored. | | | | +--------------+--------------------------------------------------------------+ +|step 5 | Two host VMs are deleted. | +| | | ++--------------+--------------------------------------------------------------+ |test verdict | Fails only if SLA is not passed, or if there is a test case | | | execution problem. | | | | diff --git a/docs/userguide/opnfv_yardstick_tc043.rst b/docs/userguide/opnfv_yardstick_tc043.rst index 59d7c6993..a873696dc 100644 --- a/docs/userguide/opnfv_yardstick_tc043.rst +++ b/docs/userguide/opnfv_yardstick_tc043.rst @@ -8,21 +8,40 @@ Yardstick Test Case Description TC043 ************************************* .. _cirros-image: https://download.cirros-cloud.net +.. _Ping: https://linux.die.net/man/8/ping +-----------------------------------------------------------------------------+ |Network Latency Between NFVI Nodes | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC043_Latency_between_NFVI_nodes | -| | measurements | +|test case id | OPNFV_YARDSTICK_TC043_LATENCY_BETWEEN_NFVI_NODES | | | | +--------------+--------------------------------------------------------------+ -|metric | RTT, Round Trip Time | +|metric | RTT (Round Trip Time) | | | | +--------------+--------------------------------------------------------------+ -|test purpose | To do a basic verification that network latency is within | -| | acceptable boundaries when packets travel between different | -| | nodes. | +|test purpose | The purpose of TC043 is to do a basic verification that | +| | network latency is within acceptable boundaries when packets | +| | travel between different NFVI nodes. | +| | | +| | The purpose is also to be able to spot the trends. | +| | Test results, graphs and similar shall be stored for | +| | comparison reasons and product evolution understanding | +| | between different OPNFV versions and/or configurations. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | ping | +| | | +| | Ping is a computer network administration software utility | +| | used to test the reachability of a host on an Internet | +| | Protocol (IP) network. It measures the round-trip time for | +| | packet sent from the originating host to a destination | +| | computer that are echoed back to the source. | +| | | ++--------------+--------------------------------------------------------------+ +|test topology | Ping packets (ICMP protocol's mandatory ECHO_REQUEST | +| | datagram) are sent from host node to target node to elicit | +| | ICMP ECHO_RESPONSE. | | | | +--------------+--------------------------------------------------------------+ |configuration | file: opnfv_yardstick_tc043.yaml | @@ -31,32 +50,33 @@ Yardstick Test Case Description TC043 | | One ping each 10 seconds. SLA RTT is set to maximum 10 ms. | | | | +--------------+--------------------------------------------------------------+ -|test tool | ping | -| | | -| | Ping is normally part of any Linux distribution, hence it | -| | doesn't need to be installed. It is also part of the | -| | Yardstick Docker image. | +|applicability | This test case can be configured with different: | | | | -+--------------+--------------------------------------------------------------+ -|references | Ping man page | +| | * packet sizes; | +| | * burst sizes; | +| | * ping intervals; | +| | * test durations; | +| | * test iterations. | | | | -| | ETSI-NFV-TST001 | +| | Default values exist. | | | | -+--------------+--------------------------------------------------------------+ -|applicability | Test case can be configured with different packet sizes, | -| | burst sizes, ping intervals and test duration. | | | SLA is optional. The SLA in this test case serves as an | -| | example. Considerably lower RTT is expected, and | -| | also normal to achieve in balanced L2 environments. However, | -| | to cover most configurations, both bare metal and fully | -| | virtualized ones, this value should be possible to achieve | -| | and acceptable for black box testing. Many real time | +| | example. Considerably lower RTT is expected, and also normal | +| | to achieve in balanced L2 environments. However, to cover | +| | most configurations, both bare metal and fully virtualized | +| | ones, this value should be possible to achieve and | +| | acceptable for black box testing. Many real time | | | applications start to suffer badly if the RTT time is higher | | | than this. Some may suffer bad also close to this RTT, while | | | others may not suffer at all. It is a compromise that may | | | have to be tuned for different configuration purposes. | | | | +--------------+--------------------------------------------------------------+ +|references | Ping_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ |pre_test | Each pod node must have ping included in it. | |conditions | | | | | @@ -64,8 +84,14 @@ Yardstick Test Case Description TC043 |test sequence | description and expected result | | | | +--------------+--------------------------------------------------------------+ -|step 1 | The pod is available. Two nodes as server and client. | -| | Ping is invoked and logs are produced and stored. | +|step 1 | Yardstick is connected with the NFVI node by using ssh. | +| | 'ping_benchmark' bash script is copyied from Jump Host to | +| | the NFVI node via the ssh tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Ping is invoked. Ping packets are sent from server node to | +| | client node. RTT results are calculated and checked against | +| | the SLA. Logs are produced and stored. | | | | | | Result: Logs are stored. | | | | diff --git a/etc/yardstick/yardstick.conf.sample b/etc/yardstick/yardstick.conf.sample index f4eff05d3..70cf71ade 100644 --- a/etc/yardstick/yardstick.conf.sample +++ b/etc/yardstick/yardstick.conf.sample @@ -8,22 +8,21 @@ ############################################################################## [DEFAULT] -# verbose = True -# debug = True -# dispatcher = http +debug = False +dispatcher = http [dispatcher_http] -# timeout = 5 -# target = http://127.0.0.1:8000/results +timeout = 5 +target = http://127.0.0.1:8000/results [dispatcher_file] -# file_path = /tmp/yardstick.out -# max_bytes = 0 -# backup_count = 0 +file_path = /tmp/yardstick.out +max_bytes = 0 +backup_count = 0 [dispatcher_influxdb] -# timeout = 5 -# target = http://127.0.0.1:8086 -# db_name = yardstick -# username = root -# password = root +timeout = 5 +target = http://127.0.0.1:8086 +db_name = yardstick +username = root +password = root diff --git a/requirements.txt b/requirements.txt index 6b4edf3f0..9c037ed79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -84,3 +84,4 @@ pyroute2==0.4.10 docker-py==1.10.6 flasgger==0.5.13 flask-restful-swagger==0.19 +SQLAlchemy==1.1.4 @@ -25,7 +25,8 @@ setup( 'resources/scripts/remove/*.bash' ], 'etc': [ - 'yardstick/nodes/*/*.yaml' + 'yardstick/nodes/*/*.yaml', + 'yardstick/*.sample' ], 'tests': [ 'opnfv/*/*.yaml', diff --git a/tests/ci/clean_images.sh b/tests/ci/clean_images.sh index 5d661283d..fa4a54df6 100755 --- a/tests/ci/clean_images.sh +++ b/tests/ci/clean_images.sh @@ -27,3 +27,10 @@ cleanup() openstack flavor delete yardstick-flavor &> /dev/null || true } + +main() +{ + cleanup +} + +main diff --git a/tests/ci/yardstick-verify b/tests/ci/yardstick-verify index 46b32cc2c..f9d98a4da 100755 --- a/tests/ci/yardstick-verify +++ b/tests/ci/yardstick-verify @@ -320,8 +320,6 @@ main() source $YARDSTICK_REPO_DIR/tests/ci/clean_images.sh - cleanup - trap "error_exit" EXIT SIGTERM source $YARDSTICK_REPO_DIR/tests/ci/load_images.sh diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py index 443b3e810..705e1ad87 100644 --- a/yardstick/common/constants.py +++ b/yardstick/common/constants.py @@ -1,3 +1,11 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## import os DOCKER_URL = 'unix://var/run/docker.sock' @@ -15,6 +23,7 @@ GRAFANA_TAGS = '3.1.1' dirname = os.path.dirname abspath = os.path.abspath +join = os.path.join sep = os.path.sep INSTALLERS = ['apex', 'compass', 'fuel', 'joid'] @@ -25,14 +34,21 @@ YARDSTICK_REPOS_DIR = '/home/opnfv/repos/yardstick' YARDSTICK_CONFIG_DIR = '/etc/yardstick/' -YARDSTICK_CONFIG_FILE = os.path.join(YARDSTICK_CONFIG_DIR, 'yardstick.conf') +YARDSTICK_CONFIG_FILE = join(YARDSTICK_CONFIG_DIR, 'yardstick.conf') + +YARDSTICK_CONFIG_SAMPLE_DIR = join(YARDSTICK_ROOT_PATH, 'etc/yardstick/') + +YARDSTICK_CONFIG_SAMPLE_FILE = join(YARDSTICK_CONFIG_SAMPLE_DIR, + 'yardstick.conf.sample') RELENG_DIR = '/home/opnfv/repos/releng' OS_FETCH_SCRIPT = 'utils/fetch_os_creds.sh' +CLEAN_IMAGES_SCRIPT = 'tests/ci/clean_images.sh' + LOAD_IMAGES_SCRIPT = 'tests/ci/load_images.sh' -OPENSTACK_RC_FILE = os.path.join(YARDSTICK_CONFIG_DIR, 'openstack.creds') +OPENSTACK_RC_FILE = join(YARDSTICK_CONFIG_DIR, 'openstack.creds') YARDSTICK_ENV_ACTION_API = 'http://localhost:5000/yardstick/env/action' |