diff options
248 files changed, 7594 insertions, 2786 deletions
diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 59dbef0ba..000000000 --- a/Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -FROM ubuntu:14.04 -MAINTAINER Hans Feldt <hans.feldt@ericsson.com> - -# TODO: Is there some easy way to get the fastest/closest mirror? -#RUN sed -i 's/archive.ubuntu.com/ftp.acc.umu.se/g' /etc/apt/sources.list - -RUN apt-get update && apt-get install -y \ - libffi-dev \ - libssl-dev \ - libxml2-dev \ - libxslt1-dev \ - python \ - python-dev \ - python-setuptools && \ - easy_install -U setuptools - -COPY . /tmp/yardstick - -RUN cd /tmp/yardstick && \ - python setup.py install && \ - apt-get -y remove \ - binutils \ - cpp \ - gcc \ - libffi-dev \ - libssl-dev \ - python3 \ - python-dev && \ - apt-get -y autoremove && \ - apt-get clean && \ - useradd -u 65500 -m yardstick && \ - cp -a samples /home/yardstick && \ - chown -R yardstick /home/yardstick/samples && \ - chgrp -R yardstick /home/yardstick/samples && \ - rm -rf /tmp/* && \ - rm -rf /var/lib/apt/lists/* - -USER yardstick -CMD bash --login -ENV HOME /home/yardstick -WORKDIR /home/yardstick @@ -12,12 +12,13 @@ Repository: yardstick Committers: jorgen.w.karlsson@ericsson.com -houjingwen@huawei.com -wenjing_chu@dell.com -liangqi1@huawei.com jean.gaoliang@huawei.com vincenzo.m.riccobene@intel.com +lvjing5@huawei.com +wu.zhihui1@zte.com.cn +14_ykl@tongji.edu.cn +limingjiang@huawei.com Link to TSC approval: http://meetbot.opnfv.org/meetings/ -Link to approval of additional submitters: +Link to approval of additional submitters: Link to approval of new PTL: Done via Condorcet Internet Voting Service, avaliable from Raymond Piak diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/__init__.py diff --git a/api/actions/__init__.py b/api/actions/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/actions/__init__.py diff --git a/api/actions/env.py b/api/actions/env.py new file mode 100644 index 000000000..78f9b72ed --- /dev/null +++ b/api/actions/env.py @@ -0,0 +1,258 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import threading +import subprocess +import time +import json +import os +import errno + +from docker import Client + +from yardstick.common import constants as config +from yardstick.common import utils as yardstick_utils +from yardstick.common.httpClient import HttpClient +from api import conf as api_conf +from api.utils import influx +from api.utils.common import result_handler + +logger = logging.getLogger(__name__) + + +def createGrafanaContainer(args): + thread = threading.Thread(target=_create_grafana) + thread.start() + return result_handler('success', []) + + +def _create_grafana(): + client = Client(base_url=config.DOCKER_URL) + + try: + if not _check_image_exist(client, '%s:%s' % (config.GRAFANA_IMAGE, + config.GRAFANA_TAGS)): + client.pull(config.GRAFANA_IMAGE, config.GRAFANA_TAGS) + + _create_grafana_container(client) + + time.sleep(5) + + _create_data_source() + + _create_dashboard() + except Exception as e: + logger.debug('Error: %s', e) + + +def _create_dashboard(): + url = 'http://admin:admin@%s:3000/api/dashboards/db' % api_conf.GATEWAY_IP + data = json.load(file('../dashboard/ping_dashboard.json')) + HttpClient().post(url, data) + + +def _create_data_source(): + url = 'http://admin:admin@%s:3000/api/datasources' % api_conf.GATEWAY_IP + data = { + "name": "yardstick", + "type": "influxdb", + "access": "proxy", + "url": "http://%s:8086" % api_conf.GATEWAY_IP, + "password": "root", + "user": "root", + "database": "yardstick", + "basicAuth": True, + "basicAuthUser": "admin", + "basicAuthPassword": "admin", + "isDefault": False, + } + HttpClient().post(url, data) + + +def _create_grafana_container(client): + ports = [3000] + port_bindings = {k: k for k in ports} + host_config = client.create_host_config(port_bindings=port_bindings) + + container = client.create_container(image='%s:%s' % (config.GRAFANA_IMAGE, + config.GRAFANA_TAGS), + ports=ports, + detach=True, + tty=True, + host_config=host_config) + client.start(container) + + +def _check_image_exist(client, t): + return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags']) + + +def createInfluxDBContainer(args): + thread = threading.Thread(target=_create_influxdb) + thread.start() + return result_handler('success', []) + + +def _create_influxdb(): + client = Client(base_url=config.DOCKER_URL) + + try: + _config_output_file() + + if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE, + config.INFLUXDB_TAG)): + client.pull(config.INFLUXDB_IMAGE, tag=config.INFLUXDB_TAG) + + _create_influxdb_container(client) + + time.sleep(5) + + _config_influxdb() + except Exception as e: + logger.debug('Error: %s', e) + + +def _create_influxdb_container(client): + + ports = [8083, 8086] + port_bindings = {k: k for k in ports} + host_config = client.create_host_config(port_bindings=port_bindings) + + container = client.create_container(image='%s:%s' % (config.INFLUXDB_IMAGE, + config.INFLUXDB_TAG), + ports=ports, + detach=True, + tty=True, + host_config=host_config) + client.start(container) + + +def _config_influxdb(): + try: + client = influx.get_data_db_client() + client.create_user(config.USER, config.PASSWORD, config.DATABASE) + client.create_database(config.DATABASE) + logger.info('Success to config influxDB') + except Exception as e: + logger.debug('Failed to config influxDB: %s', e) + + +def _config_output_file(): + yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) + with open(config.YARDSTICK_CONFIG_FILE, 'w') as f: + f.write("""\ +[DEFAULT] +debug = False +dispatcher = influxdb + +[dispatcher_file] +file_path = /tmp/yardstick.out + +[dispatcher_http] +timeout = 5 +# target = http://127.0.0.1:8000/results + +[dispatcher_influxdb] +timeout = 5 +target = http://%s:8086 +db_name = yardstick +username = root +password = root +""" + % api_conf.GATEWAY_IP) + + +def prepareYardstickEnv(args): + thread = threading.Thread(target=_prepare_env_daemon) + thread.start() + return result_handler('success', []) + + +def _prepare_env_daemon(): + + installer_ip = os.environ.get('INSTALLER_IP', 'undefined') + installer_type = os.environ.get('INSTALLER_TYPE', 'undefined') + + _check_variables(installer_ip, installer_type) + + _create_directories() + + rc_file = config.OPENSTACK_RC_FILE + + _get_remote_rc_file(rc_file, installer_ip, installer_type) + + _source_file(rc_file) + + _append_external_network(rc_file) + + # update the external_network + _source_file(rc_file) + + _load_images() + + +def _check_variables(installer_ip, installer_type): + + if installer_ip == 'undefined': + raise SystemExit('Missing INSTALLER_IP') + + if installer_type == 'undefined': + raise SystemExit('Missing INSTALLER_TYPE') + elif installer_type not in config.INSTALLERS: + raise SystemExit('INSTALLER_TYPE is not correct') + + +def _create_directories(): + yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) + + +def _source_file(rc_file): + yardstick_utils.source_env(rc_file) + + +def _get_remote_rc_file(rc_file, installer_ip, installer_type): + + os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT) + + try: + cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, + '-a', installer_ip] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + p.communicate()[0] + + if p.returncode != 0: + logger.debug('Failed to fetch credentials from installer') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def _append_external_network(rc_file): + neutron_client = yardstick_utils.get_neutron_client() + networks = neutron_client.list_networks()['networks'] + try: + ext_network = next(n['name'] for n in networks if n['router:external']) + except StopIteration: + logger.warning("Can't find external network") + else: + cmd = 'export EXTERNAL_NETWORK=%s' % ext_network + try: + with open(rc_file, 'a') as f: + f.write(cmd + '\n') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def _load_images(): + cmd = [config.LOAD_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + cwd=config.YARDSTICK_REPOS_DIR) + output = p.communicate()[0] + logger.debug('The result is: %s', output) diff --git a/api/actions/result.py b/api/actions/result.py new file mode 100644 index 000000000..1f200fbcc --- /dev/null +++ b/api/actions/result.py @@ -0,0 +1,63 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import uuid +import re + +from api.utils import influx as influx_utils +from api.utils import common as common_utils +from api import conf + +logger = logging.getLogger(__name__) + + +def getResult(args): + try: + measurement = args['measurement'] + task_id = args['task_id'] + + if re.search("[^a-zA-Z0-9_-]", measurement): + raise ValueError('invalid measurement parameter') + + uuid.UUID(task_id) + except KeyError: + message = 'measurement and task_id must be provided' + return common_utils.error_handler(message) + + query_template = "select * from %s where task_id='%s'" + query_sql = query_template % ('tasklist', task_id) + data = common_utils.translate_to_str(influx_utils.query(query_sql)) + + def _unfinished(): + return common_utils.result_handler(0, []) + + def _finished(): + query_sql = query_template % (conf.TEST_CASE_PRE + measurement, + task_id) + data = common_utils.translate_to_str(influx_utils.query(query_sql)) + if not data: + query_sql = query_template % (measurement, task_id) + data = common_utils.translate_to_str(influx_utils.query(query_sql)) + + return common_utils.result_handler(1, data) + + def _error(): + return common_utils.result_handler(2, data[0]['error']) + + try: + status = data[0]['status'] + + switcher = { + 0: _unfinished, + 1: _finished, + 2: _error + } + return switcher.get(status, lambda: 'nothing')() + except IndexError: + return common_utils.error_handler('no such task') diff --git a/api/actions/samples.py b/api/actions/samples.py new file mode 100644 index 000000000..545447aec --- /dev/null +++ b/api/actions/samples.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import os +import logging + +from api import conf +from api.utils import common as common_utils + +logger = logging.getLogger(__name__) + + +def runTestCase(args): + try: + opts = args.get('opts', {}) + testcase = args['testcase'] + except KeyError: + return common_utils.error_handler('Lack of testcase argument') + + testcase = os.path.join(conf.SAMPLE_PATH, testcase + '.yaml') + + task_id = str(uuid.uuid4()) + + command_list = ['task', 'start'] + command_list = common_utils.get_command_list(command_list, opts, testcase) + logger.debug('The command_list is: %s', command_list) + + logger.debug('Start to execute command list') + common_utils.exec_command_task(command_list, task_id) + + return common_utils.result_handler('success', task_id) diff --git a/api/actions/test.py b/api/actions/test.py new file mode 100644 index 000000000..fda0ffd32 --- /dev/null +++ b/api/actions/test.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import os +import logging + +from api import conf +from api.utils import common as common_utils + +logger = logging.getLogger(__name__) + + +def runTestCase(args): + try: + opts = args.get('opts', {}) + testcase = args['testcase'] + except KeyError: + return common_utils.error_handler('Lack of testcase argument') + + testcase = os.path.join(conf.TEST_CASE_PATH, + conf.TEST_CASE_PRE + testcase + '.yaml') + + task_id = str(uuid.uuid4()) + + command_list = ['task', 'start'] + command_list = common_utils.get_command_list(command_list, opts, testcase) + logger.debug('The command_list is: %s', command_list) + + logger.debug('Start to execute command list') + common_utils.exec_command_task(command_list, task_id) + + return common_utils.result_handler('success', task_id) diff --git a/api/api-prepare.sh b/api/api-prepare.sh new file mode 100755 index 000000000..fade8ccc6 --- /dev/null +++ b/api/api-prepare.sh @@ -0,0 +1,49 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# nginx config +nginx_config='/etc/nginx/conf.d/yardstick.conf' + +if [[ ! -e "${nginx_config}" ]];then + + cat << EOF > "${nginx_config}" +server { + listen 5000; + server_name localhost; + index index.htm index.html; + location / { + include uwsgi_params; + uwsgi_pass unix:///home/opnfv/repos/yardstick/api/yardstick.sock; + } +} +EOF +echo "daemon off;" >> /etc/nginx/nginx.conf +fi + +# nginx service start when boot +supervisor_config='/etc/supervisor/conf.d/yardstick.conf' + +if [[ ! -e "${supervisor_config}" ]];then + cat << EOF > "${supervisor_config}" +[supervisord] +nodaemon = true + +[program:yardstick_nginx] +user = root +command = service nginx restart +autorestart = true + +[program:yardstick_uwsgi] +user = root +directory = /home/opnfv/repos/yardstick/api +command = uwsgi -i yardstick.ini +autorestart = true +EOF +fi diff --git a/api/conf.py b/api/conf.py new file mode 100644 index 000000000..df44042b1 --- /dev/null +++ b/api/conf.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from pyroute2 import IPDB + + +# configuration for influxdb +with IPDB() as ip: + GATEWAY_IP = ip.routes['default'].gateway +PORT = 8086 + +TEST_ACTION = ['runTestCase'] + +TEST_CASE_PATH = '../tests/opnfv/test_cases/' + +SAMPLE_PATH = '../samples/' + +TEST_CASE_PRE = 'opnfv_yardstick_' + +TEST_SUITE_PATH = '../tests/opnfv/test_suites/' + +OUTPUT_CONFIG_FILE_PATH = '/etc/yardstick/yardstick.conf' diff --git a/api/server.py b/api/server.py new file mode 100644 index 000000000..64a2b4f96 --- /dev/null +++ b/api/server.py @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging + +from flask import Flask +from flask_restful import Api +from flasgger import Swagger + +from api.urls import urlpatterns +from yardstick import _init_logging + +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +Swagger(app) + +api = Api(app) + + +reduce(lambda a, b: a.add_resource(b.resource, b.url, + endpoint=b.endpoint) or a, urlpatterns, api) + +if __name__ == '__main__': + _init_logging() + logger.setLevel(logging.DEBUG) + logger.info('Starting server') + app.run(host='0.0.0.0') diff --git a/api/swagger/__init__.py b/api/swagger/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/swagger/__init__.py diff --git a/api/swagger/docs/results.yaml b/api/swagger/docs/results.yaml new file mode 100644 index 000000000..7bdab3eb6 --- /dev/null +++ b/api/swagger/docs/results.yaml @@ -0,0 +1,41 @@ +Query task result data + +This api offer the interface to get the result data via task_id +We will return a result json dict +--- +tags: + - Results +parameters: + - + in: query + name: action + type: string + default: getResult + required: true + - + in: query + name: measurement + type: string + description: test case name + required: true + - + in: query + name: task_id + type: string + description: the task_id you get before + required: true +responses: + 200: + description: a result json dict + schema: + id: ResultModel + properties: + status: + type: string + description: the status of the certain task + default: success + result: + schema: + type: array + items: + type: object diff --git a/api/swagger/docs/testcases.yaml b/api/swagger/docs/testcases.yaml new file mode 100644 index 000000000..7bfe5e647 --- /dev/null +++ b/api/swagger/docs/testcases.yaml @@ -0,0 +1,50 @@ +TestCases Actions
+
+This API may offer many actions, including runTestCase
+
+action: runTestCase
+This api offer the interface to run a test case in yardstick
+we will return a task_id for querying
+you can use the returned task_id to get the result data
+---
+tags:
+ - Release Action
+parameters:
+ - in: body
+ name: body
+ description: this is the input json dict
+ schema:
+ id: TestCaseActionModel
+ required:
+ - action
+ - args
+ properties:
+ action:
+ type: string
+ description: this is action for testcases
+ default: runTestCase
+ args:
+ schema:
+ id: TestCaseActionArgsModel
+ required:
+ - testcase
+ properties:
+ testcase:
+ type: string
+ description: this is the test case name
+ default: tc002
+ opts:
+ schema:
+ id: TestCaseActionArgsOptsModel
+responses:
+ 200:
+ description: A result json dict
+ schema:
+ id: result
+ properties:
+ status:
+ type: string
+ default: success
+ result:
+ type: string
+ description: task_id of this task
diff --git a/api/swagger/models.py b/api/swagger/models.py new file mode 100644 index 000000000..7c65fbbf5 --- /dev/null +++ b/api/swagger/models.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from flask_restful import fields +from flask_restful_swagger import swagger + + +# for testcases/action runTestCase action +@swagger.model +class TestCaseActionArgsOptsTaskArgModel: + resource_fields = { + } + + +@swagger.model +class TestCaseActionArgsOptsModel: + resource_fields = { + 'task-args': TestCaseActionArgsOptsTaskArgModel, + 'keep-deploy': fields.String, + 'suite': fields.String + } + + +@swagger.model +class TestCaseActionArgsModel: + resource_fields = { + 'testcase': fields.String, + 'opts': TestCaseActionArgsOptsModel + } + + +@swagger.model +class TestCaseActionModel: + resource_fields = { + 'action': fields.String, + 'args': TestCaseActionArgsModel + } + + +# for results +@swagger.model +class ResultModel: + resource_fields = { + 'status': fields.String, + 'result': fields.List + } diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 000000000..50be91ead --- /dev/null +++ b/api/urls.py @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from api import views +from api.utils.common import Url + + +urlpatterns = [ + Url('/yardstick/testcases/release/action', views.Release, 'release'), + Url('/yardstick/testcases/samples/action', views.Samples, 'samples'), + Url('/yardstick/results', views.Results, 'results'), + Url('/yardstick/env/action', views.Env, 'env') +] diff --git a/api/utils/__init__.py b/api/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/utils/__init__.py diff --git a/api/utils/common.py b/api/utils/common.py new file mode 100644 index 000000000..e3e64a72b --- /dev/null +++ b/api/utils/common.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import collections +import logging + +from flask import jsonify + +from api.utils.daemonthread import DaemonThread +from yardstick.cmd.cli import YardstickCLI + +logger = logging.getLogger(__name__) + + +def translate_to_str(object): + if isinstance(object, collections.Mapping): + return {str(k): translate_to_str(v) for k, v in object.items()} + elif isinstance(object, list): + return [translate_to_str(ele) for ele in object] + elif isinstance(object, unicode): + return str(object) + return object + + +def get_command_list(command_list, opts, args): + + command_list.append(args) + + command_list.extend(('--{}'.format(k) for k in opts if 'task-args' != k)) + + task_args = opts.get('task-args', '') + if task_args: + command_list.extend(['--task-args', str(task_args)]) + + return command_list + + +def exec_command_task(command_list, task_id): # pragma: no cover + daemonthread = DaemonThread(YardstickCLI().api, (command_list, task_id)) + daemonthread.start() + + +def error_handler(message): + logger.debug(message) + result = { + 'status': 'error', + 'message': message + } + return jsonify(result) + + +def result_handler(status, data): + result = { + 'status': status, + 'result': data + } + return jsonify(result) + + +class Url(object): + + def __init__(self, url, resource, endpoint): + super(Url, self).__init__() + self.url = url + self.resource = resource + self.endpoint = endpoint diff --git a/api/utils/daemonthread.py b/api/utils/daemonthread.py new file mode 100644 index 000000000..47c0b9108 --- /dev/null +++ b/api/utils/daemonthread.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import threading +import os +import datetime +import errno + +from api import conf +from api.utils.influx import write_data_tasklist + + +class DaemonThread(threading.Thread): + + def __init__(self, method, args): + super(DaemonThread, self).__init__(target=method, args=args) + self.method = method + self.command_list = args[0] + self.task_id = args[1] + + def run(self): + timestamp = datetime.datetime.now() + + try: + write_data_tasklist(self.task_id, timestamp, 0) + self.method(self.command_list, self.task_id) + write_data_tasklist(self.task_id, timestamp, 1) + except Exception as e: + write_data_tasklist(self.task_id, timestamp, 2, error=str(e)) + finally: + _handle_testsuite_file(self.task_id) + + +def _handle_testsuite_file(task_id): + try: + os.remove(os.path.join(conf.TEST_SUITE_PATH, task_id + '.yaml')) + except OSError as e: + if e.errno != errno.ENOENT: + raise diff --git a/api/utils/influx.py b/api/utils/influx.py new file mode 100644 index 000000000..9366ed3e9 --- /dev/null +++ b/api/utils/influx.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +from urlparse import urlsplit + +from influxdb import InfluxDBClient +import ConfigParser + +from api import conf + +logger = logging.getLogger(__name__) + + +def get_data_db_client(): + parser = ConfigParser.ConfigParser() + try: + parser.read(conf.OUTPUT_CONFIG_FILE_PATH) + dispatcher = parser.get('DEFAULT', 'dispatcher') + + if 'influxdb' != dispatcher: + raise RuntimeError + + ip = _get_ip(parser.get('dispatcher_influxdb', 'target')) + username = parser.get('dispatcher_influxdb', 'username') + password = parser.get('dispatcher_influxdb', 'password') + db_name = parser.get('dispatcher_influxdb', 'db_name') + return InfluxDBClient(ip, conf.PORT, username, password, db_name) + except ConfigParser.NoOptionError: + logger.error('can not find the key') + raise + + +def _get_ip(url): + return urlsplit(url).hostname + + +def _write_data(measurement, field, timestamp, tags): + point = { + 'measurement': measurement, + 'fields': field, + 'time': timestamp, + 'tags': tags + } + + try: + client = get_data_db_client() + + logger.debug('Start to write data: %s', point) + client.write_points([point]) + except RuntimeError: + logger.debug('dispatcher is not influxdb') + + +def write_data_tasklist(task_id, timestamp, status, error=''): + field = {'status': status, 'error': error} + tags = {'task_id': task_id} + _write_data('tasklist', field, timestamp, tags) + + +def query(query_sql): + try: + client = get_data_db_client() + logger.debug('Start to query: %s', query_sql) + return list(client.query(query_sql).get_points()) + except RuntimeError: + logger.error('dispatcher is not influxdb') + raise diff --git a/api/views.py b/api/views.py new file mode 100644 index 000000000..928d8e9eb --- /dev/null +++ b/api/views.py @@ -0,0 +1,82 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import os + +from flask import request +from flask_restful import Resource +from flasgger.utils import swag_from + +from api.utils import common as common_utils +from api.swagger import models +from api.actions import test as test_action +from api.actions import samples as samples_action +from api.actions import result as result_action +from api.actions import env as env_action + +logger = logging.getLogger(__name__) + + +TestCaseActionModel = models.TestCaseActionModel +TestCaseActionArgsModel = models.TestCaseActionArgsModel +TestCaseActionArgsOptsModel = models.TestCaseActionArgsOptsModel +TestCaseActionArgsOptsTaskArgModel = models.TestCaseActionArgsOptsTaskArgModel + + +class Release(Resource): + @swag_from(os.getcwd() + '/swagger/docs/testcases.yaml') + def post(self): + action = common_utils.translate_to_str(request.json.get('action', '')) + args = common_utils.translate_to_str(request.json.get('args', {})) + logger.debug('Input args is: action: %s, args: %s', action, args) + + try: + return getattr(test_action, action)(args) + except AttributeError: + return common_utils.error_handler('Wrong action') + + +class Samples(Resource): + def post(self): + action = common_utils.translate_to_str(request.json.get('action', '')) + args = common_utils.translate_to_str(request.json.get('args', {})) + logger.debug('Input args is: action: %s, args: %s', action, args) + + try: + return getattr(samples_action, action)(args) + except AttributeError: + return common_utils.error_handler('Wrong action') + + +ResultModel = models.ResultModel + + +class Results(Resource): + @swag_from(os.getcwd() + '/swagger/docs/results.yaml') + def get(self): + args = common_utils.translate_to_str(request.args) + action = args.get('action', '') + logger.debug('Input args is: action: %s, args: %s', action, args) + + try: + return getattr(result_action, action)(args) + except AttributeError: + return common_utils.error_handler('Wrong action') + + +class Env(Resource): + def post(self): + action = common_utils.translate_to_str(request.json.get('action', '')) + args = common_utils.translate_to_str(request.json.get('args', {})) + logger.debug('Input args is: action: %s, args: %s', action, args) + + try: + return getattr(env_action, action)(args) + except AttributeError: + return common_utils.error_handler('Wrong action') diff --git a/api/yardstick.ini b/api/yardstick.ini new file mode 100644 index 000000000..01025c2ef --- /dev/null +++ b/api/yardstick.ini @@ -0,0 +1,16 @@ +[uwsgi] +master = true +debug = true +chdir = /home/opnfv/repos/yardstick/api +module = server +plugins = python +processes = 10 +threads = 5 +async = true +max-requests = 5000 +chmod-socket = 666 +callable = app +enable-threads = true +close-on-exec = 1 +daemonize=/home/opnfv/repos/yardstick/api/uwsgi.log +socket = /home/opnfv/repos/yardstick/api/yardstick.sock diff --git a/api/yardstick.sock b/api/yardstick.sock new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/yardstick.sock diff --git a/dashboard/KVMFORNFV-Cyclictest b/dashboard/KVMFORNFV-Cyclictest new file mode 100644 index 000000000..36c1ec778 --- /dev/null +++ b/dashboard/KVMFORNFV-Cyclictest @@ -0,0 +1,517 @@ +{ + "id": 42, + "title": "KVMFORNFV-Cyclictest", + "tags": [ + "kvmfornfv-cyclictest" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "", + "panels": [ + { + "content": "<h5 style=\"font-family:Verdana\"> <a style=\"color:#31A7D3\"><center>OPNFV_KVMFORNFV_Cyclictest - Real time benchmark</center> </a></h5>\n<center>\n<p>Cyclictest is used to test the guest timer latency with idle host/guest,Host memory stress and Host CPU stress.\nFor more information see <a style=\"color:#31A7D3\"; href=\"https://wiki.opnfv.org/display/kvm/KVM4NFV+Test++Environment\">Cyclictest</a></p>\n</center>\n\n\n", + "editable": true, + "error": false, + "id": 6, + "isNew": true, + "links": [], + "mode": "html", + "span": 12, + "title": "", + "type": "text" + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "", + "panels": [ + { + "content": "", + "editable": true, + "error": false, + "id": 9, + "isNew": true, + "links": [], + "mode": "markdown", + "span": 12, + "style": {}, + "title": "Test Case Execution", + "type": "text" + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "300px", + "panels": [ + { + "aliasColors": { + "kvmfornfv_cyclictest_idle_idle.avg": "#7EB26D", + "kvmfornfv_cyclictest_idle_idle.max": "#6ED0E0", + "kvmfornfv_cyclictest_idle_idle.min": "#EAB839" + }, + "bars": false, + "datasource": "yardstick-vtc", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 10, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "kvmfornfv_cyclictest_idle_idle.min", + "yaxis": 1 + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "kvmfornfv_cyclictest_idle_idle", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "avg" + ], + "type": "field" + } + ], + [ + { + "params": [ + "max" + ], + "type": "field" + } + ], + [ + { + "params": [ + "min" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "kvmfornfv_cyclictest_idle_idle", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "µs", + "label": "Latency", + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "kvmfornfv_cyclictest_idle_idle.avg": "#7EB26D" + }, + "bars": false, + "datasource": "yardstick-vtc", + "decimals": null, + "editable": true, + "error": false, + "fill": 2, + "grid": { + "threshold1": 100, + "threshold1Color": "rgb(227, 225, 213)", + "threshold2": null, + "threshold2Color": "rgba(199, 177, 177, 0)", + "thresholdLine": true + }, + "hideTimeOverride": false, + "id": 11, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 100, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 5, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "kvmfornfv_cyclictest_idle_idle", + "policy": "default", + "query": "SELECT \"avg\" FROM \"kvmfornfv_cyclictest_idle_idle\" WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "avg" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "AVG Graph", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "µs", + "label": "Latency", + "logBase": 1024, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "kvmfornfv_cyclictest_idle_idle.min": "#EAB839" + }, + "bars": false, + "datasource": "yardstick-vtc", + "editable": true, + "error": false, + "fill": 2, + "grid": { + "threshold1": 50, + "threshold1Color": "#ebe9d9", + "threshold2": null, + "threshold2Color": "#e9d8d8", + "thresholdLine": true + }, + "id": 12, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 5, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "kvmfornfv_cyclictest_idle_idle", + "policy": "default", + "query": "SELECT \"min\" FROM \"kvmfornfv_cyclictest_idle_idle\" WHERE $timeFilter", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "min" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "MIN Graph", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "µs", + "label": "Latency", + "logBase": 1024, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "kvmfornfv_cyclictest_idle_idle.max": "#6ED0E0" + }, + "bars": false, + "datasource": "yardstick-vtc", + "editable": true, + "error": false, + "fill": 2, + "grid": { + "threshold1": 1000, + "threshold1Color": "#e9e7d6", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": true + }, + "id": 13, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 5, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "kvmfornfv_cyclictest_idle_idle", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "max" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "MAX Graph", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "µs", + "label": "Latency", + "logBase": 1024, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "refresh": "1d", + "schemaVersion": 12, + "version": 1, + "links": [], + "gnetId": null +} diff --git a/dashboard/ping_dashboard.json b/dashboard/ping_dashboard.json new file mode 100644 index 000000000..538fe065b --- /dev/null +++ b/dashboard/ping_dashboard.json @@ -0,0 +1 @@ +{"meta":{"type":"db","canSave":true,"canEdit":true,"canStar":true,"slug":null,"expires":"0001-01-01T00:00:00Z","created":"2016-10-09T00:45:46Z","updated":"2016-10-09T03:12:01Z","updatedBy":"admin","createdBy":"admin","version":7},"dashboard":{"id":null,"title":"opnfv_yardstick_tc002","tags":[],"style":"dark","timezone":"browser","editable":true,"hideControls":false,"sharedCrosshair":false,"rows":[{"title":"New row","height":"25px","editable":true,"collapse":false,"panels":[{"title":"","error":false,"span":12,"editable":true,"type":"text","isNew":true,"id":2,"mode":"html","content":"<div class=\"text-center\" style=\"padding: 10px 0 5px 0\">\n<style>\nh1 {\n\ttext-shadow: -1px -1px 1px #fff, 1px 1px 1px #31A7D3;\n\tcolor: #31A7D3;\n\topacity: 0.8;\n\tfont: 50px '31A7D3';\n}\n</style>\n<body>\n<h1>Ping Dashboard</h1>\n</body>","links":[],"height":"25"}]},{"collapse":false,"editable":true,"height":"250px","panels":[{"aliasColors":{},"bars":false,"datasource":"yardstick","editable":true,"error":false,"fill":1,"grid":{"threshold1":1,"threshold1Color":"rgba(216, 200, 27, 0.27)","threshold2":0.5,"threshold2Color":"rgba(234, 112, 112, 0.22)","thresholdLine":false},"id":1,"isNew":true,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":true,"min":true,"rightSide":false,"show":false,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","percentage":false,"pointradius":5,"points":true,"renderer":"flot","seriesOverrides":[],"span":12,"stack":false,"steppedLine":false,"targets":[{"dsType":"influxdb","groupBy":[{"params":["$interval"],"type":"time"},{"params":["null"],"type":"fill"}],"measurement":"opnfv_yardstick_tc002","policy":"default","refId":"A","resultFormat":"time_series","select":[[{"params":["rtt.ares"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]}],"timeFrom":null,"timeShift":null,"title":"","tooltip":{"msResolution":true,"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"show":true},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"title":"Row"}],"time":{"from":"now-5m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"templating":{"list":[]},"annotations":{"list":[]},"refresh":"10s","schemaVersion":12,"version":2,"links":[],"gnetId":null}} diff --git a/tests/ci/docker/yardstick-ci/Dockerfile b/docker/Dockerfile index da755d11d..23afef74e 100644 --- a/tests/ci/docker/yardstick-ci/Dockerfile +++ b/docker/Dockerfile @@ -11,14 +11,27 @@ FROM ubuntu:14.04 LABEL image=opnfv/yardstick +ARG BRANCH=master + # GIT repo directory ENV REPOS_DIR /home/opnfv/repos # Yardstick repo ENV YARDSTICK_REPO_DIR ${REPOS_DIR}/yardstick ENV RELENG_REPO_DIR ${REPOS_DIR}/releng +RUN sed -i -e 's/^deb /deb [arch=amd64] /g' /etc/apt/sources.list +RUN sed -i -e 's/^deb-src /# deb-src /g' /etc/apt/sources.list +RUN echo "\n\ +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty main universe multiverse restricted \n\ +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main universe multiverse restricted \n\ +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted \n\ +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" >> /etc/apt/sources.list +RUN echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf +RUN dpkg --add-architecture arm64 RUN apt-get update && apt-get install -y \ + qemu-user-static \ + libc6:arm64 \ wget \ expect \ curl \ @@ -32,8 +45,12 @@ RUN apt-get update && apt-get install -y \ python-dev \ libxml2-dev \ libxslt1-dev \ + nginx \ + uwsgi \ + uwsgi-plugin-python \ + supervisor \ python-setuptools && \ - easy_install -U setuptools + easy_install -U setuptools==30.0.0 RUN apt-get -y autoremove && \ apt-get clean @@ -41,15 +58,20 @@ RUN apt-get -y autoremove && \ RUN mkdir -p ${REPOS_DIR} RUN git config --global http.sslVerify false -RUN git clone https://gerrit.opnfv.org/gerrit/yardstick ${YARDSTICK_REPO_DIR} -RUN git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR} +RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/yardstick ${YARDSTICK_REPO_DIR} +RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR} # install yardstick + dependencies RUN cd ${YARDSTICK_REPO_DIR} && easy_install -U pip -RUN cd ${YARDSTICK_REPO_DIR} && pip install -r tests/ci/requirements.txt +RUN cd ${YARDSTICK_REPO_DIR} && pip install -r requirements.txt RUN cd ${YARDSTICK_REPO_DIR} && pip install . +RUN ${YARDSTICK_REPO_DIR}/api/api-prepare.sh + +EXPOSE 5000 + ADD http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img /home/opnfv/images/ ADD http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img /home/opnfv/images/ COPY ./exec_tests.sh /usr/local/bin/ +CMD ["/usr/bin/supervisord"] diff --git a/tests/ci/docker/Makefile b/docker/Makefile index 036d67db3..036d67db3 100644 --- a/tests/ci/docker/Makefile +++ b/docker/Makefile diff --git a/tests/ci/docker/yardstick-ci/exec_tests.sh b/docker/exec_tests.sh index 9aee240da..9aee240da 100755 --- a/tests/ci/docker/yardstick-ci/exec_tests.sh +++ b/docker/exec_tests.sh diff --git a/docs/release/release-notes.rst b/docs/release/release-notes.rst index bc58b2134..8df0776df 100644 --- a/docs/release/release-notes.rst +++ b/docs/release/release-notes.rst @@ -1,12 +1,19 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) OPNFV, Ericsson AB and others. +======= +License +======= + +OPNFV Colorado release note for Yardstick Docs +are licensed under a Creative Commons Attribution 4.0 International License. +You should have received a copy of the license along with this. +If not, see <http://creativecommons.org/licenses/by/4.0/>. +The *Yardstick framework*, the *Yardstick test cases* and the *ApexLake* +experimental framework are opensource software, licensed under the terms of the +Apache License, Version 2.0. -============================================ -OPNFV Brahmaputra Release Note for Yardstick -============================================ +========================================= +OPNFV Colorado Release Note for Yardstick +========================================= .. toctree:: :maxdepth: 2 @@ -21,33 +28,25 @@ OPNFV Brahmaputra Release Note for Yardstick Abstract ======== -This document compiles the release notes for the OPNFV Brahmaputra release -for Yardstick framework as well as Yardstick_ Project deliverables. - -License -======= - -The *Yardstick framework*, the *Yardstick test cases* and the *ApexLake* -experimental framework are opensource software, licensed under the terms of the -Apache License, Version 2.0. +This document describes the release note of Yardstick project. Version History =============== -+---------------+--------------------+---------------------------------+ -| *Date* | *Version* | *Comment* | -| | | | -+---------------+--------------------+---------------------------------+ -| Apr 27th,2016 | 3.0 | Brahmaputra release | -| | | | -+---------------+--------------------+---------------------------------+ -| Mar 30th,2016 | 2.0 | Brahmaputra release | -| | | | -+---------------+--------------------+---------------------------------+ -| Feb 25th,2016 | 1.0 | Brahmaputra release | -| | | | -+---------------+--------------------+---------------------------------+ ++----------------+--------------------+---------------------------------+ +| *Date* | *Version* | *Comment* | +| | | | ++----------------+--------------------+---------------------------------+ +| Dec 5th, 2016 | 3.0 | Yardstick for Colorado release | +| | | | ++----------------+--------------------+---------------------------------+ +| Oct 27th, 2016 | 2.0 | Yardstick for Colorado release | +| | | | ++----------------+--------------------+---------------------------------+ +| Aug 22nd, 2016 | 1.0 | Yardstick for Colorado release | +| | | | ++----------------+--------------------+---------------------------------+ Important Notes @@ -62,10 +61,10 @@ The *Yardstick* framework is *installer*, *infrastructure* and *application* independent. -Summary -======= +OPNFV Colorado Release +====================== -This Brahmaputra release provides *Yardstick* as a framework for NFVI testing +This Colorado release provides *Yardstick* as a framework for NFVI testing and OPNFV feature testing, automated in the OPNFV CI pipeline, including: * Documentation generated with Sphinx @@ -84,14 +83,16 @@ and OPNFV feature testing, automated in the OPNFV CI pipeline, including: * Automated Yardstick test results visualization - * Dashboard_ using Grafana (user:opnfv/password: opnfv), influxDB used as + * Dashboard_ using Grafana (user:opnfv/password: opnfv), influxDB is used as backend * Yardstick framework source code * Yardstick test cases yaml files -For Brahmaputra release, the *Yardstick framework* is used for the following +* Yardstick pliug-in configration yaml files, plug-in install/remove scripts + +For Colorado release, the *Yardstick framework* is used for the following testing: * OPNFV platform testing - generic test cases to measure the categories: @@ -112,7 +113,9 @@ testing: * Parser -* Test cases added in Brahmaputra2.0: + * StorPerf + + * VSperf * virtual Traffic Classifier @@ -132,165 +135,60 @@ Release Data | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.3.0 | +| **Repo/tag** | yardstick/colorado.3.0 | | | | +--------------------------------------+--------------------------------------+ -| **Yardstick Docker image tag** | brahmaputra.3.0 | +| **Yardstick Docker image tag** | colorado.3.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Brahmaputra | +| **Release designation** | Colorado | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Apr 27th, 2016 | +| **Release date** | December 5th, 2016 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | OPNFV Brahmaputra release | +| **Purpose of the delivery** | OPNFV Colorado release 3.0 | | | | +--------------------------------------+--------------------------------------+ -Version Change --------------- - -Module Version Changes -~~~~~~~~~~~~~~~~~~~~~~ - -This is the third tracked release of Yardstick. It is based on following -upstream versions: - -- OpenStack Liberty - -- OpenDaylight Beryllium - - -Document Version Changes -~~~~~~~~~~~~~~~~~~~~~~~~ - -This is the third tracked version of the Yardstick framework in OPNFV. -It includes the following documentation updates: - -- Yardstick User Guide: corrected faulty links - -- Yardstick Code Documentation: no changes - -- Yardstick Release Notes for Yardstick: this document - -- Test Results report for Brahmaputra testing with Yardstick: updated listed of -verified scenarios and limitations - -Documentation updates on the second tracked version: - -- Yardstick User Guide: added software architecture chapter - -- Yardstick Code Documentation: no changes - -- Yardstick Release Notes for Yardstick: this document - -- Test Results report for Brahmaputra testing with Yardstick: added test cases -and results for virtual Traffic Classifier - - -Reason for Version ------------------- - -Feature additions -~~~~~~~~~~~~~~~~~ - -No new features. - -Brahmaputra.2.0: - -+----------------------------+------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-227 | Heat HTTPS SSL support. | -| | | -+----------------------------+------------------------------------------------+ - - -Corrected Faults -~~~~~~~~~~~~~~~~ - -No corrected faults. - -Brahmaputra.2.0: +Deliverables +============ -+----------------------------+------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Change copyrights for base scenario, runners, | -| | dispatchers, cover. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Update setup.py and dependencies | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Add missing dependencies to docker file | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Fix Heat template for noisy neighbors deploy | -| | | -+----------------------------+------------------------------------------------+ +Documents +--------- -Known Faults -~~~~~~~~~~~~ + - User Guide: http://artifacts.opnfv.org/yardstick/colorado/docs/userguide/index.html + - Test Results: http://artifacts.opnfv.org/yardstick/colorado/docs/results/overview.html -+----------------------------+------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-175 | Running test suite, if a test cases running | -| | failed, the test is stopped. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-176 | Fix plotter bug since Output format has been | -| | changed. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-216 | ArgsAlreadyParsedError: arguments already | -| | parsed: cannot register CLI option. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-231 | Installation instructions on Wiki not accurate | -| | | -+----------------------------+------------------------------------------------+ - -.. note:: The faults not related to *Yardstick* framework, addressing scenarios - which were not fully verified, are listed in the OPNFV installer's release - notes. - - -Deliverables ------------- Software Deliverables -~~~~~~~~~~~~~~~~~~~~~ +--------------------- -**Yardstick framework source code <brahmaputra.3.0>** +**Yardstick framework source code <colorado.3.0>** +--------------------------------------+--------------------------------------+ | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.3.0 | +| **Repo/tag** | yardstick/colorado.3.0 | | | | +--------------------------------------+--------------------------------------+ -| **Yardstick Docker image tag** | brahmaputra.3.0 | +| **Yardstick Docker image tag** | colorado.3.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Brahmaputra | +| **Release designation** | Colorado | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Apr 27th, 2016 | +| **Release date** | December 5th, 2016 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | OPNFV Brahmaputra release | +| **Purpose of the delivery** | OPNFV Colorado release | | | | +--------------------------------------+--------------------------------------+ + **Contexts** +---------------------+-------------------------------------------------------+ @@ -326,6 +224,7 @@ Software Deliverables | | | +---------------------+-------------------------------------------------------+ + **Scenarios** +---------------------+-------------------------------------------------------+ @@ -353,15 +252,27 @@ Software Deliverables | | | | | * lmbench | | | | +| | * lmbench_cache | +| | | | | * perf | | | | | | * unixbench | | | | +| | * ramspeed | +| | | +| | * cachestat | +| | | +| | * memeoryload | +| | | +| | * computecapacity | +| | | +---------------------+-------------------------------------------------------+ | *Networking* | * iperf3 | | | | | | * netperf | | | | +| | * netperf_node | +| | | | | * ping | | | | | | * ping6 | @@ -380,13 +291,23 @@ Software Deliverables | | | | | * vtc throughput in the presence of noisy neighbors | | | | +| | * networkcapacity | +| | | +| | * netutilization | +| | | +---------------------+-------------------------------------------------------+ | *Parser* | Tosca2Heat | | | | +---------------------+-------------------------------------------------------+ | *Storage* | fio | | | | +| | storagecapacity | +| | | +---------------------+-------------------------------------------------------+ +| *StorPerf* | storperf | +| | | ++---------------------+-------------------------------------------------------+ + **API to Other Frameworks** @@ -401,6 +322,7 @@ Software Deliverables | | | +---------------------+-------------------------------------------------------+ + **Test Results Output** +-----------------------------+-----------------------------------------------+ @@ -413,13 +335,13 @@ Software Deliverables | http | Post data to html. | | | | +-----------------------------+-----------------------------------------------+ -| influxdb | Post data to influxdB. | +| influxdb | Post data to influxDB. | | | | +-----------------------------+-----------------------------------------------+ Delivered Test cases -~~~~~~~~~~~~~~~~~~~~ +-------------------- * Generic NFVI test cases @@ -427,6 +349,8 @@ Delivered Test cases * OPNFV_YARDSTICK_TCOO2 - NW Latency + * OPNFV_YARDSTICK_TCOO4 - Cache Utilization + * OPNFV_YARDSTICK_TCOO5 - Storage Performance * OPNFV_YARDSTICK_TCOO8 - Packet Loss Extended Test @@ -448,6 +372,31 @@ Delivered Test cases * OPNFV_YARDSTICK_TCO38 - Latency, CPU Load, Throughput, Packet Loss Extended Test + * OPNFV_YARDSTICK_TCO42 - Network Performance + + * OPNFV_YARDSTICK_TCO43 - Network Latency Between NFVI Nodes + + * OPNFV_YARDSTICK_TCO44 - Memory Utilization + + * OPNFV_YARDSTICK_TCO55 - Compute Capacity + + * OPNFV_YARDSTICK_TCO61 - Network Utilization + + * OPNFV_YARDSTICK_TCO63 - Storage Capacity + + * OPNFV_YARDSTICK_TCO69 - Memory Bandwidth + + * OPNFV_YARDSTICK_TCO70 - Latency, Memory Utilization, Throughput, Packet + Loss + + * OPNFV_YARDSTICK_TCO71 - Latency, Cache Utilization, Throughput, Packet Loss + + * OPNFV_YARDSTICK_TCO72 - Latency, Network Utilization, Throughput, Packet + Loss + + * OPNFV_YARDSTICK_TC073 - Network Latency and Throughput Between Nodes + + * OPNFV_YARDSTICK_TCO75 - Network Capacity and Scale * Test Cases for OPNFV HA Project: @@ -455,6 +404,34 @@ Delivered Test cases * OPNFV_YARDSTICK_TC025 - HA: OpenStacK Controller Node abnormally down + * OPNFV_YARDSTICK_TCO45 - HA: Control node Openstack service down - neutron + server + + * OPNFV_YARDSTICK_TC046 - HA: Control node Openstack service down - keystone + + * OPNFV_YARDSTICK_TCO47 - HA: Control node Openstack service down - glance + api + + * OPNFV_YARDSTICK_TC048 - HA: Control node Openstack service down - cinder + api + + * OPNFV_YARDSTICK_TCO49 - HA: Control node Openstack service down - swift + proxy + + * OPNFV_YARDSTICK_TC050 - HA: OpenStack Controller Node Network High + Availability + + * OPNFV_YARDSTICK_TCO51 - HA: OpenStack Controller Node CPU Overload High + Availability + + * OPNFV_YARDSTICK_TC052 - HA: OpenStack Controller Node Disk I/O Block High + Availability + + * OPNFV_YARDSTICK_TCO53 - HA: OpenStack Controller Load Balance Service High + Availability + + * OPNFV_YARDSTICK_TC054 - HA: OpenStack Virtual IP High Availability + * Test Case for OPNFV IPv6 Project: * OPNFV_YARDSTICK_TCO27 - IPv6 connectivity @@ -467,6 +444,10 @@ Delivered Test cases * OPNFV_YARDSTICK_TCO40 - Verify Parser Yang-to-Tosca +* Test Case for OPNFV StorPerf Project: + + * OPNFV_YARDSTICK_TCO74 - Storperf + * Test Cases for Virtual Traffic Classifier: * OPNFV_YARDSTICK_TC006 - Virtual Traffic Classifier Data Plane Throughput @@ -479,3 +460,234 @@ Benchmarking in presence of noisy neighbors Test * OPNFV_YARDSTICK_TC021 - Virtual Traffic Classifier Instantiation in presence of noisy neighbors Test + + +Version Change +============== + +Module Version Changes +---------------------- + +This is the second tracked release of Yardstick. It is based on following +upstream versions: + +- ONOS Goldeneye + +- OpenStack Mitaka + +- OpenDaylight Beryllium + + +Document Version Changes +------------------------ + +This is the second tracked version of the Yardstick framework in OPNFV. +It includes the following documentation updates: + +- Yardstick User Guide: added yardstick plugin chapter; added Store Other +Project's Test Results in InfluxDB chapter; Refine yardstick instantion chapter. + +- Yardstick Code Documentation: no changes + +- Yardstick Release Notes for Yardstick: this document + +- Test Results report for Colorado testing with Yardstick: updated listed of +verified scenarios and limitations + + +Feature additions +----------------- + - Yardstick plugin + - Yardstick reporting + - StorPerf Integration + + +Scenario Matrix +=============== + +For Colorado 3.0, Yardstick was tested on the following scenarios: + ++-------------------------+---------+---------+---------+---------+ +| Scenario | Apex | Compass | Fuel | Joid | ++=========================+=========+=========+=========+=========+ +| os-nosdn-nofeature-noha | | | | X | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-nofeature-ha | X | | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-nofeature-ha | X | X | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-nofeature-noha| | X | | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l3-nofeature-ha | X | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l3-nofeature-ha | | X | | | ++-------------------------+---------+---------+---------+---------+ +| os-onos-sfc-ha | X | | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-onos-nofeature-ha | X | | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-onos-nofeature-noha | | X | | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-sfc-ha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-sfc-noha | X | X | | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-bgpvpn-ha | X | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-bgpvpn-noha | | X | | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-kvm-ha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-kvm-noha | | X | | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-ovs-ha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-ovs-noha | X | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-ocl-nofeature-ha | | | | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-lxd-ha | | | | X | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-lxd-noha | | | | X | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-fdio-noha | X | | | | ++-------------------------+---------+---------+---------+---------+ + + +Test results +============ + +Test results are available in: + + - jenkins logs on CI: https://build.opnfv.org/ci/view/yardstick/ + +The reporting pages can be found at: + + * apex: http://testresults.opnfv.org/reporting/yardstick/release/colorado/index-status-apex.html + * compass: http://testresults.opnfv.org/reporting/yardstick/release/colorado/index-status-compass.html + * fuel: http://testresults.opnfv.org/reporting/yardstick/release/colorado/index-status-fuel.html + * joid: http://testresults.opnfv.org/reporting/yardstick/release/colorado/index-status-joid.html + +You can get additional details through test logs on http://artifacts.opnfv.org/. +As no search engine is available on the OPNFV artifact web site you must +retrieve the pod identifier on which the tests have been executed (see +field pod in any of the results) then click on the selected POD and look +for the date of the test you are interested in. + + +Known Issues/Faults +------------ + - Floating IP not supported in bgpvpn scenario + - Floating IP not supported in apex-os-odl_l3-nofeature-ha scenario + +.. note:: The faults not related to *Yardstick* framework, addressing scenarios + which were not fully verified, are listed in the OPNFV installer's release + notes. + + +Corrected Faults +---------------- + +Colorado.3.0: + ++----------------------------+------------------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-239 | Define process for working with Yardstick | +| | Grafana dashboard. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-373 | Add os-odl_l2-fdio-ha scenario support. | +| | | ++----------------------------+------------------------------------------------+ + + +Colorado.2.0: + ++----------------------------+------------------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-325 | Provide raw format yardstick vm image for | +| | nova-lxd scenario. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-358 | tc027 ipv6 test case to de-coupling to the | +| | installers. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-359 | ipv6 testcase disable port-security on | +| | vRouter. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-363 | ipv6 testcase to support fuel. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-367 | Add d3 graph presentation to yardstick | +| | reporting. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-371 | Provide raw format yardstick vm image for | +| | nova-lxd scenario. | +| | | ++----------------------------+------------------------------------------------+ +| JIRA: YARDSTICK-372 | cannot find yardstick-img-dpdk-modify and | +| | yardstick-img-lxd-modify in environment | +| | varibales. | +| | | ++----------------------------+------------------------------------------------+ + + +Colorado 3.0 known restrictions/issues +================================== ++-----------+-----------+----------------------------------------------+ +| Installer | Scenario | Issue | ++===========+===========+==============================================+ +| any | *-bgpvpn | Floating ips not supported. Some Test cases | +| | | related to floating ips are excluded. | ++-----------+-----------+----------------------------------------------+ +| any | odl_l3-* | Some test cases related to using floating IP | +| | | addresses fail because of a known ODL bug. | +| | | https://jira.opnfv.org/browse/APEX-112 | ++-----------+-----------+----------------------------------------------+ + + +Open JIRA tickets +================= + + +Useful links +============ + + - wiki project page: https://wiki.opnfv.org/display/yardstick/Yardstick + + - wiki Yardstick Colorado release planing page: https://wiki.opnfv.org/display/yardstick/Yardstick+Colorado+Release+Planning + + - wiki Yardstick Colorado release jira page: https://wiki.opnfv.org/display/yardstick/Jira+Yardstick-Colorado + + - Yardstick repo: https://git.opnfv.org/cgit/yardstick + + - Yardstick CI dashboard: https://build.opnfv.org/ci/view/yardstick + + - Yardstick grafana dashboard: http://testresults.opnfv.org/grafana/ + + - Yardstick IRC chanel: #opnfv-yardstick + +.. _`YARDSTICK-239` : https://jira.opnfv.org/browse/YARDSTICK-239 + +.. _`YARDSTICK-325` : https://jira.opnfv.org/browse/YARDSTICK-325 + +.. _`YARDSTICK-358` : https://jira.opnfv.org/browse/YARDSTICK-358 + +.. _`YARDSTICK-359` : https://jira.opnfv.org/browse/YARDSTICK-359 + +.. _`YARDSTICK-363` : https://jira.opnfv.org/browse/YARDSTICK-363 + +.. _`YARDSTICK-367` : https://jira.opnfv.org/browse/YARDSTICK-367 + +.. _`YARDSTICK-371` : https://jira.opnfv.org/browse/YARDSTICK-371 + +.. _`YARDSTICK-372` : https://jira.opnfv.org/browse/YARDSTICK-372 + +.. _`YARDSTICK-373` : https://jira.opnfv.org/browse/YARDSTICK-373 diff --git a/docs/results/apex-os-odl_l2-nofeature-ha.rst b/docs/results/apex-os-odl_l2-nofeature-ha.rst deleted file mode 100644 index 1477e2a19..000000000 --- a/docs/results/apex-os-odl_l2-nofeature-ha.rst +++ /dev/null @@ -1,106 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================ -Test Results for apex-os-odl_l2-nofeature-ha -============================================ - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Dashboard: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _POD1: https://wiki.opnfv.org/pharos_rls_b_labs - -Overview of test results ------------------------- - -See Dashboard_ for viewing test result metrics for each respective test case. - -All of the test case results below are based on scenario test runs on the -LF POD1, between February 19 and February 24. - -TC002 ------ - -The round-trip-time (RTT) between 2 VMs on different blades is measured using -ping. - -The results for the observed period show minimum 0.37ms, maximum 0.49ms, -average 0.45ms. -SLA set to 10 ms, only used as a reference; no value has yet been defined by -OPNFV. - -TC005 ------ - -The IO read bandwidth for the observed period show average between 124KB/s and -129 KB/s, with a minimum 372KB/s and maximum 448KB/s. - -SLA set to 400KB/s, only used as a reference; no value has yet been defined by -OPNFV. - -TC010 ------ - -The measurements for memory latency for various sizes and strides are shown in -Dashboard_. For 48MB, the minimum is 22.75 and maximum 30.77 ns. - -SLA set to 30 ns, only used as a reference; no value has yet been defined by -OPNFV. - -TC011 ------ - -Packet delay variation between 2 VMs on different blades is measured using -Iperf3. - -The mimimum packet delay variation measured is 2.5us and the maximum 8.6us. - -TC012 ------ - -See Dashboard_ for results. - -SLA set to 15 GB/s, only used as a reference, no value has yet been defined by -OPNFV. - -TC014 ------ - -The Unixbench processor single and parallel speed scores show scores between -3625 and 3660. - -No SLA set. - -TC037 ------ - -See Dashboard_ for results. - -Detailed test results ---------------------- - -The scenario was run on LF POD1_ with: -Apex -ODL Beryllium - - -Rationale for decisions ------------------------ - -Pass - -Tests were successfully executed and metrics collected. -No SLA was verified. To be decided on in next release of OPNFV. - -Conclusions and recommendations -------------------------------- - -Execute tests over a longer period of time, with time reference to versions of -components, for allowing better understanding of the behavior of the system. diff --git a/docs/results/apex-os-odl_l2-sfc-ha.rst b/docs/results/apex-os-odl_l2-sfc-ha.rst deleted file mode 100644 index 9a190cbbf..000000000 --- a/docs/results/apex-os-odl_l2-sfc-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -====================================== -Test Results for apex-os-odl_l2-sfc-ha -====================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/apex-os-odl_l3-nofeature-ha.rst b/docs/results/apex-os-odl_l3-nofeature-ha.rst deleted file mode 100644 index b9b30c8dc..000000000 --- a/docs/results/apex-os-odl_l3-nofeature-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================ -Test Results for apex-os-odl_l3-nofeature-ha -============================================ - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/apex-os-onos-nofeature-ha.rst b/docs/results/apex-os-onos-nofeature-ha.rst deleted file mode 100644 index 4aa195672..000000000 --- a/docs/results/apex-os-onos-nofeature-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -========================================== -Test Results for apex-os-onos-nofeature-ha -========================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/compass-os-nosdn-nofeature-ha.rst b/docs/results/compass-os-nosdn-nofeature-ha.rst deleted file mode 100644 index bc75a2c10..000000000 --- a/docs/results/compass-os-nosdn-nofeature-ha.rst +++ /dev/null @@ -1,133 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================== -Test Results for compass-os-nosdn-nofeature-ha -============================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Grafana: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _SC_POD: https://wiki.opnfv.org/pharos?&#community_test_labs - -Overview of test results ------------------------- - -See Grafana_ for viewing test result metrics for each respective test case. It -is possible to chose which specific scenarios to look at, and then to zoom in -on the details of each run test scenario as well. - -All of the test case results below are based on 5 consecutive scenario test -runs, each run on the Huawei SC_POD_ between February 13 and 18 in 2016. The -best would be to have more runs to draw better conclusions from, but these are -the only runs available at the time of OPNFV R2 release - -TC002 ------ -The round-trip-time (RTT) between 2 VMs on different blades is measured using -ping. The measurements are on average varying between 1.95 and 2.23 ms -with a first 2 - 3.27 ms RTT spike in the beginning of each run (This could be -because of normal ARP handling).SLA set to 10 ms. The SLA value is used as a -reference, it has not been defined by OPNFV. - -TC005 ------ -The IO read bandwidth look similar between different test runs, with an -average at approx. 145-162 MB/s. Within each run the results vary much, -minimum 2MB/s and maximum 712MB/s on the totality. -SLA set to 400KB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC010 ------ -The measurements for memory latency are consistent among test runs and results -in approx. 1.2 ns. The variations between runs are similar, between -1.215 and 1.278 ns. SLA set to 30 ns. The SLA value is used as -a reference, it has not been defined by OPNFV. - -TC011 ------ -For this scenario no results are available to report on. Probable reason is -an integer/floating point issue regarding how InfluxDB is populated with -result data from the test runs. - -TC012 ------ -The average measurements for memory bandwidth are consistent among most of the -different test runs at 12.98 - 16.73 GB/s. The last test run averages at -16.67 GB/s. Within each run the results vary, with minimal BW of 16.59 -GB/s and maximum of 16.71 GB/s of the totality. -SLA set to 15 GB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC014 ------ -The Unixbench processor single and parallel speed scores show similar results -at approx. 3000. The runs vary between scores 2499 and 3105. -No SLA set. - -TC027 ------ -The round-trip-time (RTT) between VM1 with ipv6 router on different blades is -measured using ping6. The measurements are consistent at approx. 4 ms. -SLA set to 30 ms.The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC037 ------ -The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs -on different blades are measured when increasing the amount of UDP flows sent -between the VMs using pktgen as packet generator tool. - -Round trip times and packet throughput between VMs are typically affected by -the amount of flows set up and result in higher RTT and less PPS -throughput. - -When running with less than 10000 flows the results are flat and consistent. -RTT is then approx. 30 ms and the number of PPS remains flat at approx. -230000 PPS. Beyond approx. 10000 flows and up to 1000000 (one million) there -is an even drop in RTT and PPS performance, eventually ending up at approx. -105-113 ms and 100000 PPS respectively. - -TC040 ------ -test purpose is to verify the function of Yang-to-Tosca in Parse, and this test -case is a weekly task, so it was triggered by manually, the result whether the -output is same with expected outcome is success -No SLA set. - -Detailed test results ---------------------- - -The scenario was run on Huawei SC_POD_ with: -Compass 1.0 -OpenStack Liberty -OVS 2.4.0 - -No SDN controller installed - -Rationale for decisions ------------------------ -Pass - -Tests were successfully executed and metrics collects (apart from TC011_). -No SLA was verified. To be decided on in next release of OPNFV. - -Conclusions and recommendations -------------------------------- - -The pktgen test configuration has a relatively large base effect on RTT in -TC037 compared to TC002, where there is no background load at all (30 ms -compared to 1 ms or less, which is more than a 3000 percentage different -in RTT results). The larger amounts of flows in TC037 generate worse -RTT results, in the magnitude of several hundreds of milliseconds. It would -be interesting to also make and compare all these measurements to completely -(optimized) bare metal machines running native Linux with all other relevant -tools available, e.g. lmbench, pktgen etc. diff --git a/docs/results/compass-os-odl_l2-nofeature-ha.rst b/docs/results/compass-os-odl_l2-nofeature-ha.rst deleted file mode 100644 index c266b1065..000000000 --- a/docs/results/compass-os-odl_l2-nofeature-ha.rst +++ /dev/null @@ -1,97 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -=============================================== -Test Results for compass-os-odl_l2-nofeature-ha -=============================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Dashboard: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _Sclara: https://wiki.opnfv.org/pharos_rls_b_labs - - -Overview of test results ------------------------- - -See Dashboard_ for viewing test result metrics for each respective test case. - -All of the test case results below are based on scenario test runs on the -Huawei Sclara_. - -TC002 ------ - -See Dashboard_ for results. -SLA set to 10 ms, only used as a reference; no value has yet been defined by -OPNFV. - -TC005 ------ - -See Dashboard_ for results. -SLA set to 400KB/s, only used as a reference; no value has yet been defined by -OPNFV. - -TC010 ------ - -See Dashboard_ for results. -SLA set to 30ns, only used as a reference; no value has yet been defined by -OPNFV. - -TC011 ------ - -See Dashboard_ for results. - - -TC012 ------ - -See Dashboard_ for results. -SLA set to 15 GB/s, only used as a reference; no value has yet been defined by -OPNFV. - - -TC014 ------ - -See Dashboard_ for results. -No SLA set. - - -TC037 ------ - -See Dashboard_ for results. - - -Detailed test results ---------------------- - -The scenario was run on Huawei Sclara_ POD with: -Compass -ODL Beryllium - -Rationale for decisions ------------------------ - -Pass - -Tests were successfully executed and metrics collected. -No SLA was verified. To be decided on in next release of OPNFV. - - -Conclusions and recommendations -------------------------------- - -Execute tests over a longer period of time, with time reference to versions of -components, for allowing better understanding of the behavior of the system. diff --git a/docs/results/compass-os-onos-nofeature-ha.rst b/docs/results/compass-os-onos-nofeature-ha.rst deleted file mode 100644 index 8c90b9873..000000000 --- a/docs/results/compass-os-onos-nofeature-ha.rst +++ /dev/null @@ -1,97 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================= -Test Results for compass-os-onos-nofeature-ha -============================================= - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Dashboard: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _Sclara: https://wiki.opnfv.org/pharos_rls_b_labs - - -verview of test results ------------------------- - -See Dashboard_ for viewing test result metrics for each respective test case. - -All of the test case results below are based on scenario test runs on the -Huawei Sclara_. - -TC002 ------ - -See Dashboard_ for results. -SLA set to 10 ms, only used as a reference; no value has yet been defined by -OPNFV. - -TC005 ------ - -See Dashboard_ for results. -SLA set to 400KB/s, only used as a reference; no value has yet been defined by -OPNFV. - -TC010 ------ - -See Dashboard_ for results. -SLA set to 30ns, only used as a reference; no value has yet been defined by -OPNFV. - -TC011 ------ - -See Dashboard_ for results. - - -TC012 ------ - -See Dashboard_ for results. -SLA set to 15 GB/s, only used as a reference; no value has yet been defined by -OPNFV. - - -TC014 ------ - -See Dashboard_ for results. -No SLA set. - - -TC037 ------ - -See Dashboard_ for results. - - -Detailed test results ---------------------- - -The scenario was run on Huawei Sclara_ POD with: -Compass -ONOS - -Rationale for decisions ------------------------ - -Pass - -Tests were successfully executed and metrics collected. -No SLA was verified. To be decided on in next release of OPNFV. - - -Conclusions and recommendations -------------------------------- - -Execute tests over a longer period of time, with time reference to versions of -components, for allowing better understanding of the behavior of the system. diff --git a/docs/results/fuel-os-nosdn-kvm-ha.rst b/docs/results/fuel-os-nosdn-kvm-ha.rst deleted file mode 100644 index 217bab7c0..000000000 --- a/docs/results/fuel-os-nosdn-kvm-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -===================================== -Test Results for fuel-os-nosdn-kvm-ha -===================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/fuel-os-nosdn-nofeature-ha.rst b/docs/results/fuel-os-nosdn-nofeature-ha.rst deleted file mode 100644 index eb8b14741..000000000 --- a/docs/results/fuel-os-nosdn-nofeature-ha.rst +++ /dev/null @@ -1,131 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -=========================================== -Test Results for fuel-os-nosdn-nofeature-ha -=========================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Grafana: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs - -Overview of test results ------------------------- - -See Grafana_ for viewing test result metrics for each respective test case. It -is possible to chose which specific scenarios to look at, and then to zoom in -on the details of each run test scenario as well. - -All of the test case results below are based on 5 consecutive scenario test -runs, each run on the Ericsson POD2_ between February 13 and 18 in 2016. The -best would be to have more runs to draw better conclusions from, but these are -the only runs available at the time of OPNFV R2 release. - -TC002 ------ -The round-trip-time (RTT) between 2 VMs on different blades is measured using -ping. The measurements are on average varying between 0.5 and 1.1 ms -with a first 2 - 2.5 ms RTT spike in the beginning of each run (This could be -because of normal ARP handling). The 2 last runs are very similar in their -results. But, to be able to draw any further conclusions more runs should be -made. There is one measurement taken on February 16 that does not have the -first RTT spike, and less variations to the RTT. The reason for this is -unknown. There is a discussion on another test measurement made Feb. 16 in -TC037_. -SLA set to 10 ms. The SLA value is used as a reference, it has not -been defined by OPNFV. - -TC005 ------ -The IO read bandwidth look similar between different test runs, with an -average at approx. 160-170 MB/s. Within each run the results vary much, -minimum 2 MB/s and maximum 630 MB/s on the totality. Most runs have a -minimum of 3 MB/s (one run at 2 MB/s). The maximum BW varies much more in -absolute numbers, between 566 and 630 MB/s. -SLA set to 400 MB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC010 ------ -The measurements for memory latency are consistent among test runs and results -in approx. 1.2 ns. The variations between runs are similar, between -1.215 and 1.219 ns. One exception is February 16, where the varation is -greater, between 1.22 and 1.28 ns. SLA set to 30 ns. The SLA value is used as -a reference, it has not been defined by OPNFV. - -TC011 ------ -For this scenario no results are available to report on. Probable reason is -an integer/floating point issue regarding how InfluxDB is populated with -result data from the test runs. - -TC012 ------ -The average measurements for memory bandwidth are consistent among most of the -different test runs at 17.2 - 17.3 GB/s. The very first test run averages at -17.7 GB/s. Within each run the results vary, with a minimal BW of 15.4 -GB/s and maximum of 18.2 GB/s of the totality. -SLA set to 15 GB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC014 ------ -The Unixbench processor single and parallel speed scores show similar results -at approx. 3200. The runs vary between scores 3160 and 3240. -No SLA set. - -TC037 ------ -The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs -on different blades are measured when increasing the amount of UDP flows sent -between the VMs using pktgen as packet generator tool. - -Round trip times and packet throughput between VMs are typically affected by -the amount of flows set up and result in higher RTT and less PPS -throughput. - -When running with less than 10000 flows the results are flat and consistent. -RTT is then approx. 30 ms and the number of PPS remains flat at approx. -250000 PPS. Beyond approx. 10000 flows and up to 1000000 (one million) there -is an even drop in RTT and PPS performance, eventually ending up at approx. -150-250 ms and 40000 PPS respectively. - -There is one measurement made February 16 that has slightly worse results -compared to the other 4 measurements. The reason for this is unknown. For -instance anyone being logged onto the POD can be of relevance for such a -disturbance. - -Detailed test results ---------------------- -The scenario was run on Ericsson POD2_ with: -Fuel 8.0 -OpenStack Liberty -OVS 2.3.1 - -No SDN controller installed - -Rationale for decisions ------------------------ -Pass - -Tests were successfully executed and metrics collects (apart from TC011_). -No SLA was verified. To be decided on in next release of OPNFV. - -Conclusions and recommendations -------------------------------- -The pktgen test configuration has a relatively large base effect on RTT in -TC037 compared to TC002, where there is no background load at all (30 ms -compared to 1 ms or less, which is more than a 3000 percentage different -in RTT results). The larger amounts of flows in TC037 generate worse -RTT results, in the magnitude of several hundreds of milliseconds. It would -be interesting to also make and compare all these measurements to completely -(optimized) bare metal machines running native Linux with all other relevant -tools available, e.g. lmbench, pktgen etc. diff --git a/docs/results/fuel-os-nosdn-ovs-ha.rst b/docs/results/fuel-os-nosdn-ovs-ha.rst deleted file mode 100644 index 9e6e5a458..000000000 --- a/docs/results/fuel-os-nosdn-ovs-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -===================================== -Test Results for fuel-os-nosdn-ovs-ha -===================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/fuel-os-odl_l2-nofeature-ha.rst b/docs/results/fuel-os-odl_l2-nofeature-ha.rst deleted file mode 100644 index 914781684..000000000 --- a/docs/results/fuel-os-odl_l2-nofeature-ha.rst +++ /dev/null @@ -1,156 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================ -Test Results for fuel-os-odl_l2-nofeature-ha -============================================ - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Grafana: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs - -Overview of test results ------------------------- - -See Grafana_ for viewing test result metrics for each respective test case. It -is possible to chose which specific scenarios to look at, and then to zoom in -on the details of each run test scenario as well. - -All of the test case results below are based on 6 scenario test -runs, each run on the Ericsson POD2_ between February 13 and 24 in 2016. Test -case TC011_ is the greater exception for which there are only 2 test runs -available, due to earlier problems with InfluxDB test result population. -The best would be to have more runs to draw better conclusions from, but these -are the only runs available at the time of OPNFV R2 release. - -TC002 ------ -The round-trip-time (RTT) between 2 VMs on different blades is measured using -ping. Most test run measurements result on average between 0.3 and 0.5 ms, but -one date (Feb. 23) sticks out with an RTT average of 1 ms. -A few runs start with a 1 - 2 ms RTT spike (This could be because of normal ARP -handling). One test run has a greater RTT spike of 3.9 ms, which is the same -one with the 0.9 ms average. The other runs have no similar spike at all. -To be able to draw conclusions more runs should be made. -SLA set to 10 ms. The SLA value is used as a reference, it has not -been defined by OPNFV. - -TC005 ------ -The IO read bandwidth looks similar between different dates, with an -average between approx. 165 and 185 MB/s. Within each test run the results -vary, with a minimum 2 MB/s and maximum 617 MB/s on the totality. Most runs -have a minimum BW of 3 MB/s (two runs at 2 MB/s). The maximum BW varies more in -absolute numbers between the dates, between 566 and 617 MB/s. -SLA set to 400 MB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC010 ------ -The measurements for memory latency are similar between test dates and result -in approx. 1.2 ns. The variations within each test run are similar, between -1.215 and 1.219 ns. One exception is February 16, where the average is 1.222 -and varies between 1.22 and 1.28 ns. -SLA set to 30 ns. The SLA value is used as a reference, it has not been defined -by OPNFV. - -TC011 ------ -Only 2 test runs are available to report results on. - -Packet delay variation between 2 VMs on different blades is measured using -Iperf3. On the first date the reported packet delay variation varies between -0.0025 and 0.011 ms, with an average delay variation of 0.0067 ms. -On the second date the delay variation varies between 0.002 and 0.006 ms, with -an average delay variation of 0.004 ms. - -TC012 ------ -Results are reported for 5 test runs. It is not known why the 6:th test run -is missing. -Between test dates the average measurements for memory bandwidth vary between -17.4 and 17.9 GB/s. Within each test run the results vary more, with a minimal -BW of 16.4 GB/s and maximum of 18.2 GB/s on the totality. -SLA set to 15 GB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC014 ------ -Results are reported for 5 test runs. It is not known why the 6:th test run -is missing. -The Unixbench processor test run results vary between scores 3080 and 3240, -one result each date. The average score on the total is 3150. -No SLA set. - -TC037 ------ -Results are reported for 5 test runs. It is not currently known why the 6:th -test run is missing. -The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs -on different blades are measured when increasing the amount of UDP flows sent -between the VMs using pktgen as packet generator tool. - -Round trip times and packet throughput between VMs can typically be affected by -the amount of flows set up and result in higher RTT and less PPS throughput. - -The RTT results are similar throughout the different test dates and runs at -approx. 15 ms. Some test runs show an increase with many flows, in the range -towards 16 to 17 ms. One exception standing out is Feb. 15 where the average -RTT is stable at approx. 13 ms. The PPS results are not as consistent as the -RTT results. -In some test runs when running with less than approx. 10000 flows the PPS -throughput is normally flatter compared to when running with more flows, after -which the PPS throughput decreases. Around 20 percent decrease in the worst -case. For the other test runs there is however no significant change to the PPS -throughput when the number of flows are increased. In some test runs the PPS -is also greater with 1000000 flows compared to other test runs where the PPS -result is less with only 2 flows. - -The average PPS throughput in the different runs varies between 414000 and -452000 PPS. The total amount of packets in each test run is approx. 7500000 to -8200000 packets. One test run Feb. 15 sticks out with a PPS average of -558000 and approx. 1100000 packets in total (same as the on mentioned earlier -for RTT results). - -There are lost packets reported in most of the test runs. There is no observed -correlation between the amount of flows and the amount of lost packets. -The lost amount of packets normally range between 100 and 1000 per test run, -but there are spikes in the range of 10000 lost packets as well, and even -more in a rare cases. - -Detailed test results ---------------------- -The scenario was run on Ericsson POD2_ with: -Fuel 8.0 -OpenStack Liberty -OpenVirtualSwitch 2.3.1 -OpenDayLight Beryllium - -Rationale for decisions ------------------------ -Pass - -Tests were successfully executed and metrics collected. -No SLA was verified. To be decided on in next release of OPNFV. - -Conclusions and recommendations -------------------------------- -The pktgen test configuration has a relatively large base effect on RTT in -TC037 compared to TC002, where there is no background load at all. Approx. -15 ms compared to approx. 0.5 ms, which is more than a 3000 percentage -difference in RTT results. -Especially RTT and throughput come out with better results than for instance -the *fuel-os-nosdn-nofeature-ha* scenario does. The reason for this should -probably be further analyzed and understood. Also of interest could be -to make further analyzes to find patterns and reasons for lost traffic. -Also of interest could be to see if there are continuous variations where -some test cases stand out with better or worse results than the general test -case. diff --git a/docs/results/fuel-os-odl_l3-nofeature-ha.rst b/docs/results/fuel-os-odl_l3-nofeature-ha.rst deleted file mode 100644 index 7c9c377cb..000000000 --- a/docs/results/fuel-os-odl_l3-nofeature-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================ -Test Results for fuel-os-odl_l3-nofeature-ha -============================================ - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/fuel-os-onos-nofeature-ha.rst b/docs/results/fuel-os-onos-nofeature-ha.rst deleted file mode 100644 index 4e7345820..000000000 --- a/docs/results/fuel-os-onos-nofeature-ha.rst +++ /dev/null @@ -1,146 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -========================================== -Test Results for fuel-os-onos-nofeature-ha -========================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Grafana: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs - -Overview of test results ------------------------- - -See Grafana_ for viewing test result metrics for each respective test case. It -is possible to chose which specific scenarios to look at, and then to zoom in -on the details of each run test scenario as well. - -All of the test case results below are based on 7 scenario test -runs, each run on the Ericsson POD2_ between February 13 and 21 in 2016. Test -case TC011_ is not reported on due to an InfluxDB issue. -The best would be to have more runs to draw better conclusions from, but these -are the only runs available at the time of OPNFV R2 release. - -TC002 ------ -The round-trip-time (RTT) between 2 VMs on different blades is measured using -ping. The majority (5) of the test run measurements result in an average -between 0.4 and 0.5 ms. The other 2 dates stick out with an RTT average of 0.9 -to 1 ms. -The majority of the runs start with a 1 - 1.5 ms RTT spike (This could be -because of normal ARP handling). One test run has a greater RTT spike of 4 ms, -which is the same one with the 1 ms RTT average. The other runs have no similar -spike at all. To be able to draw conclusions more runs should be made. -SLA set to 10 ms. The SLA value is used as a reference, it has not -been defined by OPNFV. - -TC005 ------ -The IO read bandwidth looks similar between different dates, with an -average between approx. 170 and 185 MB/s. Within each test run the results -vary, with a minimum of 2 MB/s and maximum of 690MB/s on the totality. Most -runs have a minimum BW of 3 MB/s (one run at 2 MB/s). The maximum BW varies -more in absolute numbers between the dates, between 560 and 690 MB/s. -SLA set to 400 MB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC010 ------ -The measurements for memory latency are similar between test dates and result -in a little less average than 1.22 ns. The variations within each test run are -similar, between 1.213 and 1.226 ns. One exception is the first date, where the -average is 1.223 and varies between 1.215 and 1.275 ns. -SLA set to 30 ns. The SLA value is used as a reference, it has not been defined -by OPNFV. - -TC011 ------ -For this scenario no results are available to report on. Reason is an -integer/floating point issue regarding how InfluxDB is populated with -result data from the test runs. The issue was fixed but not in time to produce -input for this report. - -TC012 ------ -Between test dates the average measurements for memory bandwidth vary between -17.1 and 18.1 GB/s. Within each test run the results vary more, with a minimal -BW of 15.5 GB/s and maximum of 18.2 GB/s on the totality. -SLA set to 15 GB/s. The SLA value is used as a reference, it has not been -defined by OPNFV. - -TC014 ------ -The Unixbench processor test run results vary between scores 3100 and 3260, -one result each date. The average score on the total is 3170. -No SLA set. - -TC037 ------ -The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs -on different blades are measured when increasing the amount of UDP flows sent -between the VMs using pktgen as packet generator tool. - -Round trip times and packet throughput between VMs can typically be affected by -the amount of flows set up and result in higher RTT and less PPS throughput. - -There seems to be mainly two result types. One type a high and flatter -PPS throughput not very much affected by the number of flows. Here also the -average RTT is stable around 13 ms throughout all the test runs. - -The second type starts with a slightly lower PPS in the beginning than type -one, and decreases even further when passing approx. 10000 flows. Here also the -average RTT tends to start at approx. 15 ms ending with an average of 17 to 18 -ms with the maximum amount of flows running. - -Result type one can with the maximum amount of flows have a greater PPS than -the second type with the minimum amount of flows. - -For result type one the average PPS throughput in the different runs varies -between 399000 and 447000 PPS. The total amount of packets in each test run -is between approx. 7000000 and 10200000 packets. -The second result type has a PPS average of between 602000 and 621000 PPS and a -total packet amount between 10900000 and 13500000 packets. - -There are lost packets reported in many of the test runs. There is no observed -correlation between the amount of flows and the amount of lost packets. -The lost amount of packets normally range between 100 and 1000 per test run, -but there are spikes in the range of 10000 lost packets as well, and even -more in a rare cases. Some case is in the range of one million lost packets. - -Detailed test results ---------------------- -The scenario was run on Ericsson POD2_ with: -Fuel 8.0 -OpenStack Liberty -OpenVirtualSwitch 2.3.1 -OpenNetworkOperatingSystem Drake - -Rationale for decisions ------------------------ -Pass - -Tests were successfully executed and metrics collected. -No SLA was verified. To be decided on in next release of OPNFV. - -Conclusions and recommendations -------------------------------- -The pktgen test configuration has a relatively large base effect on RTT in -TC037 compared to TC002, where there is no background load at all. Approx. -15 ms compared to approx. 0.5 ms, which is more than a 3000 percentage -difference in RTT results. -Especially RTT and throughput come out with better results than for instance -the *fuel-os-nosdn-nofeature-ha* scenario does. The reason for this should -probably be further analyzed and understood. Also of interest could be -to make further analyzes to find patterns and reasons for lost traffic. -Also of interest could be to see why there are variations in some test cases, -especially visible in TC037. - diff --git a/docs/results/index.rst b/docs/results/index.rst index b828d1426..2b67f1b22 100644 --- a/docs/results/index.rst +++ b/docs/results/index.rst @@ -3,12 +3,12 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Ericsson AB and others. -==================================================== -Yardstick Test Results for OPNFV Brahmaputra Release -==================================================== +====================== +Yardstick test results +====================== .. toctree:: - :maxdepth: 2 + :maxdepth: 4 - overview.rst - results.rst +.. include:: ./overview.rst +.. include:: ./results.rst diff --git a/docs/results/joid-os-nosdn-nofeature-ha.rst b/docs/results/joid-os-nosdn-nofeature-ha.rst deleted file mode 100644 index 504f635a0..000000000 --- a/docs/results/joid-os-nosdn-nofeature-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -=========================================== -Test Results for joid-os-nosdn-nofeature-ha -=========================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/joid-os-odl_l2-nofeature-ha.rst b/docs/results/joid-os-odl_l2-nofeature-ha.rst deleted file mode 100644 index ff2890876..000000000 --- a/docs/results/joid-os-odl_l2-nofeature-ha.rst +++ /dev/null @@ -1,97 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -============================================ -Test Results for joid-os-odl_l2-nofeature-ha -============================================ - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. _Dashboard: http://130.211.154.108/grafana/dashboard/db/yardstick-main -.. _POD2: https://wiki.opnfv.org/pharos_rls_b_labs - - -Overview of test results ------------------------- - -See Dashboard_ for viewing test result metrics for each respective test case. - -All of the test case results below are based on scenario test runs on the -Orange POD2, between February 23 and February 24. - -TC002 ------ - -See Dashboard_ for results. -SLA set to 10 ms, only used as a reference; no value has yet been defined by -OPNFV. - -TC005 ------ - -See Dashboard_ for results. -SLA set to 400KB/s, only used as a reference; no value has yet been defined by -OPNFV. - -TC010 ------ - -Not executed, missing in the test suite used in the POD during the observed -period. - -TC011 ------ - -Not executed, missing in the test suite used in the POD during the observed -period. - - -TC012 ------ - -Not executed, missing in the test suite used in the POD during the observed -period. - - -TC014 ------ - -Not executed, missing in the test suite used in the POD during the observed -period. - - -TC037 ------ - -See Dashboard_ for results. - - -Detailed test results ---------------------- - -The scenario was run on Orange POD2_ with: -Joid -ODL Beryllium - -Rationale for decisions ------------------------ - -Pass - -Most tests were successfully executed and metrics collected, the non-execution -of above-mentioned tests was due to test cases missing in the Jenkins Job used -in the POD, during the observed period. -No SLA was verified. To be decided on in next release of OPNFV. - -Conclusions and recommendations -------------------------------- - -Execute tests over a longer period of time, with time reference to versions of -components, for allowing better understanding of the behavior of the system. diff --git a/docs/results/joid-os-onos-nofeature-ha.rst b/docs/results/joid-os-onos-nofeature-ha.rst deleted file mode 100644 index 13fabc465..000000000 --- a/docs/results/joid-os-onos-nofeature-ha.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - - -========================================== -Test Results for joid-os-onos-nofeature-ha -========================================== - -.. toctree:: - :maxdepth: 2 - - -Details -======= - -.. after this doc is filled, remove all comments and include the scenario in -.. results.rst by removing the comment on the file name. - - -Overview of test results ------------------------- - -.. general on metrics collected, number of iterations - -Detailed test results ---------------------- - -.. info on lab, installer, scenario - -Rationale for decisions ------------------------ -.. result analysis, pass/fail - -Conclusions and recommendations -------------------------------- - -.. did the expected behavior occured? diff --git a/docs/results/os-nosdn-kvm-ha.rst b/docs/results/os-nosdn-kvm-ha.rst new file mode 100644 index 000000000..a8a56f80e --- /dev/null +++ b/docs/results/os-nosdn-kvm-ha.rst @@ -0,0 +1,270 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +================================ +Test Results for os-nosdn-kvm-ha +================================ + +.. toctree:: + :maxdepth: 2 + + +fuel +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test +runs, each run on the Ericsson POD2_ or LF POD2_ between August 24 and 30 in +2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 0.44 and 0.75 ms. +A few runs start with a 0.65 - 0.68 ms RTT spike (This could be because of +normal ARP handling). One test run has a greater RTT spike of 1.49 ms. +To be able to draw conclusions more runs should be made. SLA set to 10 ms. +The SLA value is used as a reference, it has not been defined by OPNFV. + +TC005 +----- +The IO read bandwidth looks similar between different dates, with an +average between approx. 92 and 204 MB/s. Within each test run the results +vary, with a minimum 2 MB/s and maximum 819 MB/s on the totality. Most runs +have a minimum BW of 3 MB/s (one run at 2 MB/s). The maximum BW varies more in +absolute numbers between the dates, between 238 and 819 MB/s. +SLA set to 400 MB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC010 +----- +The measurements for memory latency are similar between test dates and result +in approx. 2.07 ns. The variations within each test run are similar, between +1.41 and 3.53 ns. +SLA set to 30 ns. The SLA value is used as a reference, it has not been defined +by OPNFV. + +TC011 +----- +Packet delay variation between 2 VMs on different blades is measured using +Iperf3. The reported packet delay variation varies between 0.0051 and 0.0243 ms, +with an average delay variation between 0.0081 ms and 0.0195 ms. + +TC012 +----- +Between test dates, the average measurements for memory bandwidth result in +approx. 13.6 GB/s. Within each test run the results vary more, with a minimal +BW of 6.09 GB/s and maximum of 16.47 GB/s on the totality. +SLA set to 15 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC014 +----- +The Unixbench processor test run results vary between scores 2316 and 3619, +one result each date. +No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +CPU utilization statistics are collected during UDP flows sent between the VMs +using pktgen as packet generator tool. The average measurements for CPU +utilization ratio vary between 1% to 2%. The peak of CPU utilization ratio +appears around 7%. + +TC069 +----- +Between test dates, the average measurements for memory bandwidth vary between +22.6 and 29.1 GB/s. Within each test run the results vary more, with a minimal +BW of 20.0 GB/s and maximum of 29.5 GB/s on the totality. +SLA set to 6 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Memory utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for memory +utilization vary between 225MB to 246MB. The peak of memory utilization appears +around 340MB. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Cache utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for cache +utilization vary between 205MB to 212MB. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. Total number of packets received per +second was average on 200 kpps and total number of packets transmitted per +second was average on 600 kpps. + +Detailed test results +--------------------- +The scenario was run on Ericsson POD2_ and LF POD2_ with: +Fuel 9.0 +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + +Conclusions and recommendations +------------------------------- +The pktgen test configuration has a relatively large base effect on RTT in +TC037 compared to TC002, where there is no background load at all. Approx. +15 ms compared to approx. 0.5 ms, which is more than a 3000 percentage +difference in RTT results. +Especially RTT and throughput come out with better results than for instance +the *fuel-os-nosdn-nofeature-ha* scenario does. The reason for this should +probably be further analyzed and understood. Also of interest could be +to make further analyzes to find patterns and reasons for lost traffic. +Also of interest could be to see if there are continuous variations where +some test cases stand out with better or worse results than the general test +case. + diff --git a/docs/results/os-nosdn-nofeature-ha.rst b/docs/results/os-nosdn-nofeature-ha.rst new file mode 100644 index 000000000..9e52731d5 --- /dev/null +++ b/docs/results/os-nosdn-nofeature-ha.rst @@ -0,0 +1,492 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +====================================== +Test Results for os-nosdn-nofeature-ha +====================================== + +.. toctree:: + :maxdepth: 2 + + +apex +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD1: https://wiki.opnfv.org/pharos?&#community_test_labs + + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test +runs, each run on the LF POD1_ between August 25 and 28 in +2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 0.74 and 1.08 ms. +A few runs start with a 0.99 - 1.07 ms RTT spike (This could be because of +normal ARP handling). One test run has a greater RTT spike of 1.35 ms. +To be able to draw conclusions more runs should be made. SLA set to 10 ms. +The SLA value is used as a reference, it has not been defined by OPNFV. + +TC005 +----- +The IO read bandwidth looks similar between different dates, with an +average between approx. 128 and 136 MB/s. Within each test run the results +vary, with a minimum 5 MB/s and maximum 446 MB/s on the totality. Most runs +have a minimum BW of 5 MB/s (one run at 6 MB/s). The maximum BW varies more in +absolute numbers between the dates, between 416 and 446 MB/s. +SLA set to 400 MB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC010 +----- +The measurements for memory latency are similar between test dates and result +in approx. 1.09 ns. The variations within each test run are similar, between +1.0860 and 1.0880 ns. +SLA set to 30 ns. The SLA value is used as a reference, it has not been defined +by OPNFV. + +TC011 +----- +Packet delay variation between 2 VMs on different blades is measured using +Iperf3. The reported packet delay variation varies between 0.0025 and 0.0148 ms, +with an average delay variation between 0.0056 ms and 0.0157 ms. + +TC012 +----- +Between test dates, the average measurements for memory bandwidth result in +approx. 19.70 GB/s. Within each test run the results vary more, with a minimal +BW of 18.16 GB/s and maximum of 20.13 GB/s on the totality. +SLA set to 15 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC014 +----- +The Unixbench processor test run results vary between scores 3224.4 and 3842.8, +one result each date. The average score on the total is 3659.5. +No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +CPU utilization statistics are collected during UDP flows sent between the VMs +using pktgen as packet generator tool. The average measurements for CPU +utilization ratio vary between 1% to 2%. The peak of CPU utilization ratio +appears around 7%. + +TC069 +----- +Between test dates, the average measurements for memory bandwidth vary between +22.6 and 29.1 GB/s. Within each test run the results vary more, with a minimal +BW of 20.0 GB/s and maximum of 29.5 GB/s on the totality. +SLA set to 6 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Memory utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for memory +utilization vary between 225MB to 246MB. The peak of memory utilization appears +around 340MB. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Cache utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for cache +utilization vary between 205MB to 212MB. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. Total number of packets received per +second was average on 200 kpps and total number of packets transmitted per +second was average on 600 kpps. + +Detailed test results +--------------------- +The scenario was run on LF POD1_ with: +Apex +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + + +Joid +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD5: https://wiki.opnfv.org/pharos?&#community_test_labs + + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Intel POD5_ between September 11 and 14 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 1.59 and 1.70 ms. +Two test runs have reached the same greater RTT spike of 3.06 ms, which are +1.66 and 1.70 ms average, but only one has the lower RTT of 1.35 ms. The other +two runs have no similar spike at all. To be able to draw conclusions more runs +should be made. SLA set to be 10 ms. The SLA value is used as a reference, it +has not been defined by OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput and the +greatest IO read bandwidth of the four runs is 173.3 MB/s. The IO read +bandwidth of the four runs looks similar on different four days, with an +average between 32.7 and 60.4 MB/s. One of the runs has a minimum BW of 429 +KM/s and other has a maximum BW of 173.3 MB/s. The SLA of read bandwidth sets +to be 400 MB/s, which is used as a reference, and it has not been defined by +OPNFV. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the four runs is 1.1 ns on average. The +variations within each test run are different, some vary from a large range and +others have a small change. For example, the largest change is on September 14, +the memory read latency of which is ranging from 1.12 ns to 1.22 ns. However, +the results on September 12 change very little, which range from 1.14 ns to +1.17 ns. The SLA sets to be 30 ns. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC011 +----- +Iperf3 is a tool for evaluating the pocket delay variation between 2 VMs on +different blades. The reported pocket delay variations of the four test runs +differ from each other. The results on September 13 within the date look +similar and the values are between 0.0087 and 0.0190 ms, which is 0.0126 ms on +average. However, on the fourth day, the pocket delay variation has a large +wide change within the date, which ranges from 0.0032 ms to 0.0121 ms and has +the minimum average value. The pocket delay variations of other two test runs +look relatively similar, which are 0.0076 ms and 0.0152 ms on average. The SLA +value sets to be 10 ms. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. Among the four test runs, the memory +bandwidth within the second day almost keep stable, which is 11.58 GB/s on +average. And the memory bandwidth of the fourth day look similar as that of the +second day, both of which remain stable. The other two test runs relatively +change from a large wide range, in which the minimum memory bandwidth is 11.22 +GB/s and the maximum bandwidth is 16.65 GB/s with an average bandwidth of about +12.20 GB/s. Here SLA set to be 15 GB/s. The SLA value is used as a reference, +it has not been defined by OPNFV. + +TC014 +----- +The Unixbench is used to measure processing speed, that is instructions per +second. It can be seen from the dashboard that the processing test results +vary from scores 3272 to 3444, and there is only one result one date. The +overall average score is 3371. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the four test runs is 119.85, 128.02, 121.40 and +126.08 kpps, of which the result of the second is the highest. The RTT results +of all the test runs keep flat at approx. 37 ms. It is obvious that the PPS +results are not as consistent as the RTT results. + +The No. flows of the four test runs are 240 k on average and the PPS results +look a little waved since the largest packet throughput is 184 kpps and the +minimum throughput is 49 K respectively. + +There are no errors of packets received in the four runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the four +runs have the similar average vaue, that is 38 ms, of which the worest RTT is +93 ms on Sep. 14th. + +CPU load of the four test runs have a large change, since the minimum value and +the peak of CPU load is 0 percent and 51 percent respectively. And the best +result is obtained on Sep. 14th. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test, which +rangs from 22.3 GB/s to 26.8 GB/s and then to 18.5 GB/s on average. Since the +test id is one, it is that only the INT memory write bandwidth is tested. On +the whole, when the block size is 8 kb and 16 kb, the memory write bandwidth +look similar with a minimal BW of 22.5 GB/s and peak value of 28.7 GB/s. SLA +sets to be 7 GB/s. The SLA value is used as a a reference, it has not been +defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT can reach +more than 80 ms and the average RTT is usually approx. 38 ms. On the whole, the +average RTTs of the four runs keep flat. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 268 MiB on Sep +14, which also has the largest minimum memory. Besides, the rest three test +runs have the similar used memory. On the other hand, the free memory of the +four runs have the same smallest minimum value, that is about 223 MiB, and the +maximum free memory of three runs have the similar result, that is 337 MiB, +except that on Sep. 14th, whose maximum free memory is 254 MiB. On the whole, +all the test runs have similar average free memory. + +Network throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +network throughput of the four test runs seem quite different, ranging from +119.85 kpps to 128.02 kpps. The average number of flows in these tests is +24000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding packet throughput +differ between 49.4k and 193.3k with an average packet throughput of approx. +125k. On the whole, the PPS results seem consistent. Within each test run of +the four runs, when number of flows becomes larger, the packet throughput seems +not larger in the meantime. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT can reach +more than 94 ms and the average RTT is usually approx. 35 ms. On the whole, the +average RTTs of the four runs keep flat. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool.The largest +cache size is 212 MiB in the four runs, and the smallest cache size is 75 MiB. +On the whole, the average cache size of the four runs is approx. 208 MiB. +Meanwhile, the tread of the buffer size looks similar with each other. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs seem quite different, ranging from 119.85 kpps to 128.02 +kpps. The average number of flows in these tests is 239.7k, and each run has a +minimum number of flows of 2 and a maximum number of flows of 1.001 Mil. At the +same time, the corresponding packet throughput differ between 49.4k and 193.3k +with an average packet throughput of approx. 125k. On the whole, the PPS results +seem consistent. Within each test run of the four runs, when number of flows +becomes larger, the packet throughput seems not larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 32 ms. The PPS results are not as consistent as the RTT results. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second differs from each other, in which the smallest number of +packets transmitted per second is 6 pps on Sep. 12ed and the largest of that is +210.8 kpps. Meanwhile, the largest total number of packets received per second +differs from each other, in which the smallest number of packets received per +second is 2 pps on Sep. 13rd and the largest of that is 250.2 kpps. + +In some test runs when running with less than approx. 90000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 1000000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on Intel POD5_ with: +Joid +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + + diff --git a/docs/results/os-nosdn-nofeature-noha.rst b/docs/results/os-nosdn-nofeature-noha.rst new file mode 100644 index 000000000..8b7c184bb --- /dev/null +++ b/docs/results/os-nosdn-nofeature-noha.rst @@ -0,0 +1,259 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +======================================== +Test Results for os-nosdn-nofeature-noha +======================================== + +.. toctree:: + :maxdepth: 2 + + +Joid +===== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD5: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Intel POD5_ between September 12 and 15 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 1.50 and 1.68 ms. +Only one test run has reached greatest RTT spike of 2.92 ms, which has +the smallest RTT of 1.06 ms. The other three runs have no similar spike at all, +the minimum and average RTTs of which are approx. 1.50 ms and 1.68 ms. SLA set to +be 10 ms. The SLA value is used as a reference, it has not been defined by +OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput, which is +measured by fio and the greatest IO read bandwidth of the four runs is 177.5 +MB/s. The IO read bandwidth of the four runs looks similar on different four +days, with an average between 46.7 and 62.5 MB/s. One of the runs has a minimum +BW of 680 KM/s and other has a maximum BW of 177.5 MB/s. The SLA of read +bandwidth sets to be 400 MB/s, which is used as a reference, and it has not +been defined by OPNFV. + +The results of storage IOPS for the four runs look similar with each other. The +test runs all have an approx. 1.55 K/s for IO reading with an minimum value of +less than 60 times per second. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the four runs is between 1.134 ns and 1.227 +ns on average. The variations within each test run are quite different, some +vary from a large range and others have a small change. For example, the +largest change is on September 15, the memory read latency of which is ranging +from 1.116 ns to 1.393 ns. However, the results on September 12 change very +little, which mainly keep flat and range from 1.124 ns to 1.55 ns. The SLA sets +to be 30 ns. The SLA value is used as a reference, it has not been defined by +OPNFV. + +TC011 +----- +Iperf3 is a tool for evaluating the pocket delay variation between 2 VMs on +different blades. The reported pocket delay variations of the four test runs +differ from each other. The results on September 13 within the date look +similar and the values are between 0.0213 and 0.0225 ms, which is 0.0217 ms on +average. However, on the third day, the packet delay variation has a large +wide change within the date, which ranges from 0.008 ms to 0.0225 ms and has +the minimum value. On Sep. 12, the packet delay is quite long, for the value is +between 0.0236 and 0.0287 ms and it also has the maximum packet delay of 0.0287 +ms. The packet delay of the last test run is 0.0151 ms on average. The SLA +value sets to be 10 ms. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. Among the four test runs, the memory +bandwidth of three test runs almost keep stable within each run, which is +11.65, 11.57 and 11.64 GB/s on average. However, the memory read and write +bandwidth on Sep. 14 has a large range, for it ranges from 11.36 GB/s to 16.68 +GB/s. Here SLA set to be 15 GB/s. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC014 +----- +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single cpu running and parallel running. It can be seen from the +dashboard that the processing test results vary from scores 3222 to 3585, and +there is only one result one date. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the four test runs is 124.8, 160.1, 113.8 and +137.3 kpps, of which the result of the second is the highest. The RTT results +of all the test runs keep flat at approx. 37 ms. It is obvious that the PPS +results are not as consistent as the RTT results. + +The No. flows of the four test runs are 240 k on average and the PPS results +look a little waved since the largest packet throughput is 243.1 kpps and the +minimum throughput is 37.6 kpps respectively. + +There are no errors of packets received in the four runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the four +runs have the similar average vaue, that is between 32 ms and 41 ms, of which +the worest RTT is 155 ms on Sep. 14th. + +CPU load is measured by mpstat, and CPU load of the four test runs seem a +little similar, since the minimum value and the peak of CPU load is between 0 +percent and 9 percent respectively. And the best result is obtained on Sep. +15th, with an CPU load of nine percent. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test, which +rangs from 22.4 GB/s to 26.5 GB/s and then to 18.6 GB/s on average. Since the +test id is one, it is that only the INT memory write bandwidth is tested. On +the whole, when the block size is 8 kb and 16 kb, the memory write bandwidth +look similar with a minimal BW of 22.5 GB/s and peak value of 28.7 GB/s. And +then with the block size becoming larger, the memory write bandwidth tends to +decrease. SLA sets to be 7 GB/s. The SLA value is used as a a reference, it has +not been defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of three test runs look +similar with each other, and Within these test runs, the maximum RTT can reach +95 ms and the average RTT is usually approx. 36 ms. The network latency tested +on Sep. 14 shows that it has a peak latency of 155 ms. But on the whole, the +average RTTs of the four runs keep flat. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 270 MiB on Sep +13, which also has the smallest minimum memory utilization. Besides, the rest +three test runs have the similar used memory with an average memory usage of +264 MiB. On the other hand, the free memory of the four runs have the same +smallest minimum value, that is about 223 MiB, and the maximum free memory of +three runs have the similar result, that is 226 MiB, except that on Sep. 13th, +whose maximum free memory is 273 MiB. On the whole, all the test runs have +similar average free memory. + +Network throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +network throughput of the four test runs seem quite different, ranging from +119.85 kpps to 128.02 kpps. The average number of flows in these tests is +240000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding packet throughput +differ between 38k and 243k with an average packet throughput of approx. 134k. +On the whole, the PPS results seem consistent. Within each test run of the four +runs, when number of flows becomes larger, the packet throughput seems not +larger in the meantime. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT can reach +79 ms and the average RTT is usually approx. 35 ms. On the whole, the average +RTTs of the four runs keep flat. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool.The largest +cache size is 214 MiB in the four runs, and the smallest cache size is 100 MiB. +On the whole, the average cache size of the four runs is approx. 210 MiB. +Meanwhile, the tread of the buffer size looks similar with each other. On the +other hand, the mean buffer size of the four runs keep flat, since they have a +minimum value of approx. 7 MiB and a maximum value of 8 MiB, with an average +value of about 8 MiB. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs seem quite different, ranging from 113.8 kpps to 124.8 kpps. +The average number of flows in these tests is 240k, and each run has a minimum +number of flows of 2 and a maximum number of flows of 1.001 Mil. At the same +time, the corresponding packet throughput differ between 47.6k and 243.1k with +an average packet throughput between 113.8k and 160.1k. Within each test run of +the four runs, when number of flows becomes larger, the packet throughput seems +not larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs +between 0 ms and 79 ms with an average leatency of approx. 35 ms. The PPS +results are not as consistent as the RTT results, for the mean packet +throughput of the four runs differ from 113.8 kpps to 124.8 kpps. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second look similar on the first three runs with a minimum +number of 10 pps and a maximum number of 97 kpps, except the one on Sep. 15th, +in which the number of packets transmitted per second is 10 pps. Meanwhile, the +largest total number of packets received per second differs from each other, +in which the smallest number of packets received per second is 1 pps and the +largest of that is 276 kpps. + +In some test runs when running with less than approx. 90000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 1000000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on Intel POD5_ with: +Joid +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. diff --git a/docs/results/os-odl_l2-bgpvpn-ha.rst b/docs/results/os-odl_l2-bgpvpn-ha.rst new file mode 100644 index 000000000..2bd6dc35d --- /dev/null +++ b/docs/results/os-odl_l2-bgpvpn-ha.rst @@ -0,0 +1,53 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +==================================== +Test Results for os-odl_l2-bgpvpn-ha +==================================== + +.. toctree:: + :maxdepth: 2 + + +fuel +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Ericsson POD2_ between September 7 and 11 in 2016. + +TC043 +----- +The round-trip-time (RTT) between 2 nodes is measured using +ping. Most test run measurements result on average between 0.21 and 0.28 ms. +A few runs start with a 0.32 - 0.35 ms RTT spike (This could be because of +normal ARP handling). To be able to draw conclusions more runs should be made. +SLA set to 10 ms. The SLA value is used as a reference, it has not been defined +by OPNFV. + +Detailed test results +--------------------- +The scenario was run on Ericsson POD2_ with: +Fuel 9.0 +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + diff --git a/docs/results/os-odl_l2-nofeature-ha.rst b/docs/results/os-odl_l2-nofeature-ha.rst new file mode 100644 index 000000000..ac0c5bb59 --- /dev/null +++ b/docs/results/os-odl_l2-nofeature-ha.rst @@ -0,0 +1,743 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +======================================= +Test Results for os-odl_l2-nofeature-ha +======================================= + +.. toctree:: + :maxdepth: 2 + + +apex +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD1: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the LF POD1_ between September 14 and 17 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 0.49 ms and 0.60 ms. +Only one test run has reached greatest RTT spike of 0.93 ms. Meanwhile, the +smallest network latency is 0.33 ms, which is obtained on Sep. 14th. +SLA set to be 10 ms. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput, which is +measured by fio and the greatest IO read bandwidth of the four runs is 416 +MB/s. The IO read bandwidth of all four runs looks similar, with an average +between 128 and 131 MB/s. One of the runs has a minimum BW of 497 KB/s. The SLA +of read bandwidth sets to be 400 MB/s, which is used as a reference, and it has +not been defined by OPNFV. + +The results of storage IOPS for the four runs look similar with each other. The +IO read times per second of the four test runs have an average value at 1k per +second, and meanwhile, the minimum result is only 45 times per second. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the four runs is between 1.0859 ns and +1.0869 ns on average. The variations within each test run are quite different, +some vary from a large range and others have a small change. For example, the +largest change is on September 14th, the memory read latency of which is ranging +from 1.091 ns to 1.086 ns. However. +The SLA sets to be 30 ns. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC011 +----- +Packet delay variation between 2 VMs on different blades is measured using +Iperf3. On the first two test runs the reported packet delay variation varies between +0.0037 and 0.0740 ms, with an average delay variation between 0.0096 ms and 0.0321. +On the second date the delay variation varies between 0.0063 and 0.0096 ms, with +an average delay variation of 0.0124 - 0.0141 ms. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. Among the four test runs, the trend of +three memory bandwidth almost look similar, which all have a narrow range, and +the average result is 19.88 GB/s. Here SLA set to be 15 GB/s. The SLA value is +used as a reference, it has not been defined by OPNFV. + +TC014 +----- +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single cpu running and parallel running. It can be seen from the +dashboard that the processing test results vary from scores 3754k to 3831k, and +there is only one result one date. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the four test runs is between 307.3 kpps and +447.1 kpps, of which the result of the third run is the highest. The RTT +results of all the test runs keep flat at approx. 15 ms. It is obvious that the +PPS results are not as consistent as the RTT results. + +The No. flows of the four test runs are 240 k on average and the PPS results +look a little waved since the largest packet throughput is 418.1 kpps and the +minimum throughput is 326.5 kpps respectively. + +There are no errors of packets received in the four runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the four +runs have the similar average vaue, that is approx. 15 ms. + +CPU load is measured by mpstat, and CPU load of the four test runs seem a +little similar, since the minimum value and the peak of CPU load is between 0 +percent and nine percent respectively. And the best result is obtained on Sep. +1, with an CPU load of nine percent. But on the whole, the CPU load is very +poor, since the average value is quite small. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test, which +rangs from 28.2 GB/s to 29.5 GB/s and then to 29.2 GB/s on average. Since the +test id is one, it is that only the INT memory write bandwidth is tested. On +the whole, when the block size is 2 kb or 16 kb, the memory write bandwidth +look similar with a minimal BW of 25.8 GB/s and peak value of 28.3 GB/s. And +then with the block size becoming larger, the memory write bandwidth tends to +decrease. SLA sets to be 7 GB/s. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other, and within these test runs, the maximum RTT can +reach 39 ms and the average RTT is usually approx. 15 ms. The network latency +tested on Sep. 1 and Sep. 8 have a peak latency of 39 ms. But on the whole, +the average RTTs of the five runs keep flat and the network latency is +relatively short. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 267 MiB for the +four runs. In general, the four test runs have very large memory utilization, +which can reach 257 MiB on average. On the other hand, for the mean free memory, +the four test runs have the similar trend with that of the mean used memory. +In general, the mean free memory change from 233 MiB to 241 MiB. + +Packet throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +packet throughput of the four test runs seem quite different, ranging from +305.3 kpps to 447.1 kpps. The average number of flows in these tests is +240000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding average packet +throughput is between 354.4 kpps and 381.8 kpps. In summary, the PPS results +seem consistent. Within each test run of the four runs, when number of flows +becomes larger, the packet throughput seems not larger at the same time. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT is only 42 +ms and the average RTT is usually approx. 15 ms. On the whole, the average +RTTs of the four runs keep stable and the network latency is relatively small. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool. The largest +cache size is 212 MiB, which is same for the four runs, and the smallest cache +size is 75 MiB. On the whole, the average cache size of the four runs look the +same and is between 197 MiB and 211 MiB. Meanwhile, the tread of the buffer +size keep flat, since they have a minimum value of 7 MiB and a maximum value of +8 MiB, with an average value of about 7.9 MiB. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs differ from 354.4 kpps to 381.8 kpps. The average number of +flows in these tests is 240k, and each run has a minimum number of flows of 2 +and a maximum number of flows of 1.001 Mil. At the same time, the corresponding +packet throughput differ between 305.3 kpps to 447.1 kpps. Within each test run +of the four runs, when number of flows becomes larger, the packet throughput +seems not larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs +between 0 ms and 42 ms with an average leatency of less than 15 ms. The PPS +results are not as consistent as the RTT results, for the mean packet +throughput of the four runs differ from 354.4 kpps to 381.8 kpps. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second look similar for three test runs, whose values change a +lot from 10 pps to 501 kpps. While results of the rest test run seem the same +and keep stable with the average number of packets transmitted per second of 10 +pps. However, the total number of packets received per second of the four runs +look similar, which have a large wide range of 2 pps to 815 kpps. + +In some test runs when running with less than approx. 251000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 251000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on LF POD1_ with: +Apex +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + + + +fuel +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Ericsson POD2_ or LF POD2_ between August 25 and 29 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 0.5 and 0.6 ms. +A few runs start with a 1 - 1.5 ms RTT spike (This could be because of normal ARP +handling). One test run has a greater RTT spike of 1.9 ms, which is the same +one with the 0.7 ms average. The other runs have no similar spike at all. +To be able to draw conclusions more runs should be made. +SLA set to 10 ms. The SLA value is used as a reference, it has not +been defined by OPNFV. + +TC005 +----- +The IO read bandwidth looks similar between different dates, with an +average between approx. 170 and 200 MB/s. Within each test run the results +vary, with a minimum 2 MB/s and maximum 838 MB/s on the totality. Most runs +have a minimum BW of 3 MB/s (two runs at 2 MB/s). The maximum BW varies more in +absolute numbers between the dates, between 617 and 838 MB/s. +SLA set to 400 MB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC010 +----- +The measurements for memory latency are similar between test dates and result +in approx. 1.2 ns. The variations within each test run are similar, between +1.215 and 1.219 ns. One exception is February 16, where the average is 1.222 +and varies between 1.22 and 1.28 ns. +SLA set to 30 ns. The SLA value is used as a reference, it has not been defined +by OPNFV. + +TC011 +----- +Packet delay variation between 2 VMs on different blades is measured using +Iperf3. On the first date the reported packet delay variation varies between +0.0025 and 0.011 ms, with an average delay variation of 0.0067 ms. +On the second date the delay variation varies between 0.002 and 0.006 ms, with +an average delay variation of 0.004 ms. + +TC012 +----- +Between test dates, the average measurements for memory bandwidth vary between +17.4 and 17.9 GB/s. Within each test run the results vary more, with a minimal +BW of 16.4 GB/s and maximum of 18.2 GB/s on the totality. +SLA set to 15 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC014 +----- +The Unixbench processor test run results vary between scores 3080 and 3240, +one result each date. The average score on the total is 3150. +No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +CPU utilization statistics are collected during UDP flows sent between the VMs +using pktgen as packet generator tool. The average measurements for CPU +utilization ratio vary between 1% to 2%. The peak of CPU utilization ratio +appears around 7%. + +TC069 +----- +Between test dates, the average measurements for memory bandwidth vary between +15.5 and 25.4 GB/s. Within each test run the results vary more, with a minimal +BW of 9.7 GB/s and maximum of 29.5 GB/s on the totality. +SLA set to 6 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Memory utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for memory +utilization vary between 225MB to 246MB. The peak of memory utilization appears +around 340MB. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Cache utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for cache +utilization vary between 205MB to 212MB. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. Total number of packets received per +second was average on 200 kpps and total number of packets transmitted per +second was average on 600 kpps. + +Detailed test results +--------------------- +The scenario was run on Ericsson POD2_ and LF POD2_ with: +Fuel 9.0 +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + +Conclusions and recommendations +------------------------------- +The pktgen test configuration has a relatively large base effect on RTT in +TC037 compared to TC002, where there is no background load at all. Approx. +15 ms compared to approx. 0.5 ms, which is more than a 3000 percentage +difference in RTT results. +Especially RTT and throughput come out with better results than for instance +the *fuel-os-nosdn-nofeature-ha* scenario does. The reason for this should +probably be further analyzed and understood. Also of interest could be +to make further analyzes to find patterns and reasons for lost traffic. +Also of interest could be to see if there are continuous variations where +some test cases stand out with better or worse results than the general test +case. + + + +Joid +===== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD6: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Intel POD6_ between September 1 and 8 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 1.01 ms and 1.88 ms. +Only one test run has reached greatest RTT spike of 1.88 ms. Meanwhile, the +smallest network latency is 1.01 ms, which is obtained on Sep. 1st. In general, +the average of network latency of the four test runs are between 1.29 ms and +1.34 ms. SLA set to be 10 ms. The SLA value is used as a reference, it has not +been defined by OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput, which is +measured by fio and the greatest IO read bandwidth of the four runs is 183.65 +MB/s. The IO read bandwidth of the three runs looks similar, with an average +between 62.9 and 64.3 MB/s, except one on Sep. 1, for its maximum storage +throughput is only 159.1 MB/s. One of the runs has a minimum BW of 685 KB/s and +other has a maximum BW of 183.6 MB/s. The SLA of read bandwidth sets to be +400 MB/s, which is used as a reference, and it has not been defined by OPNFV. + +The results of storage IOPS for the four runs look similar with each other. The +IO read times per second of the four test runs have an average value between +1.41k per second and 1.64k per second, and meanwhile, the minimum result is +only 55 times per second. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the four runs is between 1.152 ns and 1.179 +ns on average. The variations within each test run are quite different, some +vary from a large range and others have a small change. For example, the +largest change is on September 8, the memory read latency of which is ranging +from 1.120 ns to 1.221 ns. However, the results on September 7 change very +little. The SLA sets to be 30 ns. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC011 +----- +Iperf3 is a tool for evaluating the packet delay variation between 2 VMs on +different blades. The reported packet delay variations of the four test runs +differ from each other. In general, the packet delay of the first two runs look +similar, for they both stay stable within each run. And the mean packet delay +of them are 0.0087 ms and 0.0127 ms respectively. Of the four runs, the fourth +has the worst result, because the packet delay reaches 0.0187 ms. The SLA value +sets to be 10 ms. The SLA value is used as a reference, it has not been defined +by OPNFV. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. Among the four test runs, the trend of +three memory bandwidth almost look similar, which all have a narrow range, and +the average result is 11.78 GB/s. Here SLA set to be 15 GB/s. The SLA value is +used as a reference, it has not been defined by OPNFV. + +TC014 +----- +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single cpu running and parallel running. It can be seen from the +dashboard that the processing test results vary from scores 3260k to 3328k, and +there is only one result one date. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the four test runs is between 307.3 kpps and +447.1 kpps, of which the result of the third run is the highest. The RTT +results of all the test runs keep flat at approx. 15 ms. It is obvious that the +PPS results are not as consistent as the RTT results. + +The No. flows of the four test runs are 240 k on average and the PPS results +look a little waved since the largest packet throughput is 418.1 kpps and the +minimum throughput is 326.5 kpps respectively. + +There are no errors of packets received in the four runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the four +runs have the similar average vaue, that is approx. 15 ms. + +CPU load is measured by mpstat, and CPU load of the four test runs seem a +little similar, since the minimum value and the peak of CPU load is between 0 +percent and nine percent respectively. And the best result is obtained on Sep. +1, with an CPU load of nine percent. But on the whole, the CPU load is very +poor, since the average value is quite small. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test, which +rangs from 21.9 GB/s to 25.9 GB/s and then to 17.8 GB/s on average. Since the +test id is one, it is that only the INT memory write bandwidth is tested. On +the whole, when the block size is 2 kb or 16 kb, the memory write bandwidth +look similar with a minimal BW of 24.8 GB/s and peak value of 27.8 GB/s. And +then with the block size becoming larger, the memory write bandwidth tends to +decrease. SLA sets to be 7 GB/s. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other, and within these test runs, the maximum RTT can +reach 39 ms and the average RTT is usually approx. 15 ms. The network latency +tested on Sep. 1 and Sep. 8 have a peak latency of 39 ms. But on the whole, +the average RTTs of the five runs keep flat and the network latency is +relatively short. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 267 MiB for the +four runs. In general, the four test runs have very large memory utilization, +which can reach 257 MiB on average. On the other hand, for the mean free memory, +the four test runs have the similar trend with that of the mean used memory. +In general, the mean free memory change from 233 MiB to 241 MiB. + +Packet throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +packet throughput of the four test runs seem quite different, ranging from +305.3 kpps to 447.1 kpps. The average number of flows in these tests is +240000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding average packet +throughput is between 354.4 kpps and 381.8 kpps. In summary, the PPS results +seem consistent. Within each test run of the four runs, when number of flows +becomes larger, the packet throughput seems not larger at the same time. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT is only 42 +ms and the average RTT is usually approx. 15 ms. On the whole, the average +RTTs of the four runs keep stable and the network latency is relatively small. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool. The largest +cache size is 212 MiB, which is same for the four runs, and the smallest cache +size is 75 MiB. On the whole, the average cache size of the four runs look the +same and is between 197 MiB and 211 MiB. Meanwhile, the tread of the buffer +size keep flat, since they have a minimum value of 7 MiB and a maximum value of +8 MiB, with an average value of about 7.9 MiB. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs differ from 354.4 kpps to 381.8 kpps. The average number of +flows in these tests is 240k, and each run has a minimum number of flows of 2 +and a maximum number of flows of 1.001 Mil. At the same time, the corresponding +packet throughput differ between 305.3 kpps to 447.1 kpps. Within each test run +of the four runs, when number of flows becomes larger, the packet throughput +seems not larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs +between 0 ms and 42 ms with an average leatency of less than 15 ms. The PPS +results are not as consistent as the RTT results, for the mean packet +throughput of the four runs differ from 354.4 kpps to 381.8 kpps. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second look similar for three test runs, whose values change a +lot from 10 pps to 501 kpps. While results of the rest test run seem the same +and keep stable with the average number of packets transmitted per second of 10 +pps. However, the total number of packets received per second of the four runs +look similar, which have a large wide range of 2 pps to 815 kpps. + +In some test runs when running with less than approx. 251000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 251000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on Intel POD6_ with: +Joid +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + diff --git a/docs/results/os-odl_l2-sfc-ha.rst b/docs/results/os-odl_l2-sfc-ha.rst new file mode 100644 index 000000000..e27562cae --- /dev/null +++ b/docs/results/os-odl_l2-sfc-ha.rst @@ -0,0 +1,231 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +================================== +Test Results for os-odl_l2-sfc-ha +================================== + +.. toctree:: + :maxdepth: 2 + + +Fuel +===== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the LF POD2_ or Ericsson POD2_ between September 16 and 20 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 0.32 ms and 1.42 ms. +Only one test run on Sep. 20 has reached greatest RTT spike of 4.66 ms. +Meanwhile, the smallest network latency is 0.16 ms, which is obtained on Sep. +17th. To sum up, the curve of network latency has very small wave, which is +less than 5 ms. SLA sets to be 10 ms. The SLA value is used as a reference, it +has not been defined by OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput, which is +measured by fio and the greatest IO read bandwidth of the four runs is 734 +MB/s. The IO read bandwidth of the first three runs looks similar, with an +average of less than 100 KB/s, except one on Sep. 20, whose maximum storage +throughput can reach 734 MB/s. The SLA of read bandwidth sets to be 400 MB/s, +which is used as a reference, and it has not been defined by OPNFV. + +The results of storage IOPS for the four runs look similar with each other. The +IO read times per second of the four test runs have an average value between +1.8k per second and 3.27k per second, and meanwhile, the minimum result is +only 60 times per second. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the four runs is between 1.085 ns and 1.218 +ns on average. The variations within each test run are quite small. For +Ericsson pod2, the average of memory latency is approx. 1.217 ms. While for LF +pod2, the average value is about 1.085 ms. It can be seen that the performance +of LF is better than Ericsson's. The SLA sets to be 30 ns. The SLA value is +used as a reference, it has not been defined by OPNFV. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. The four test runs all have a narrow range +of change with the average memory and write BW of 18.5 GB/s. Here SLA set to be +15 GB/s. The SLA value is used as a reference, it has not been defined by OPNFV. + +TC014 +----- +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single cpu running and parallel running. It can be seen from the +dashboard that the processing test results vary from scores 3209k to 3843k, and +there is only one result one date. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the three test runs is between 439 kpps and +582 kpps, and the test run on Sep. 17th has the lowest average value of 371 +kpps. The RTT results of all the test runs keep flat at approx. 10 ms. It is +obvious that the PPS results are not as consistent as the RTT results. + +The No. flows of the four test runs are 240 k on average and the PPS results +look a little waved, since the largest packet throughput is 680 kpps and the +minimum throughput is 319 kpps respectively. + +There are no errors of packets received in the four runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the four +runs have the similar trend of RTT with the average value of approx. 12 ms. + +CPU load is measured by mpstat, and CPU load of the four test runs seem a +little similar, since the minimum value and the peak of CPU load is between 0 +percent and ten percent respectively. And the best result is obtained on Sep. +17th, with an CPU load of ten percent. But on the whole, the CPU load is very +poor, since the average value is quite small. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the average memory write +bandwidth tends to become larger first and then smaller within every run test +for the two pods, which rangs from 25.1 GB/s to 29.4 GB/s and then to 19.2 GB/s +on average. Since the test id is one, it is that only the INT memory write +bandwidth is tested. On the whole, with the block size becoming larger, the +memory write bandwidth tends to decrease. SLA sets to be 7 GB/s. The SLA value +is used as a reference, it has not been defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other, and within these test runs, the maximum RTT can +reach 27 ms and the average RTT is usually approx. 12 ms. The network latency +tested on Sep. 27th has a peak latency of 27 ms. But on the whole, the average +RTTs of the four runs keep flat. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 269 MiB for the +four runs. In general, the four test runs have very large memory utilization, +which can reach 251 MiB on average. On the other hand, for the mean free memory, +the four test runs have the similar trend with that of the mean used memory. +In general, the mean free memory change from 231 MiB to 248 MiB. + +Packet throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +packet throughput of the four test runs seem quite different, ranging from +371 kpps to 582 kpps. The average number of flows in these tests is +240000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding average packet +throughput is between 319 kpps and 680 kpps. In summary, the PPS results +seem consistent. Within each test run of the four runs, when number of flows +becomes larger, the packet throughput seems not larger at the same time. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT is only 24 +ms and the average RTT is usually approx. 12 ms. On the whole, the average +RTTs of the four runs keep stable and the network latency is relatively small. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool. The largest +cache size is 213 MiB, and the smallest cache size is 99 MiB, which is same for +the four runs. On the whole, the average cache size of the four runs look the +same and is between 184 MiB and 205 MiB. Meanwhile, the tread of the buffer +size keep stable, since they have a minimum value of 7 MiB and a maximum value of +8 MiB. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs differ from 371 kpps to 582 kpps. The average number of +flows in these tests is 240k, and each run has a minimum number of flows of 2 +and a maximum number of flows of 1.001 Mil. At the same time, the corresponding +packet throughput differ between 319 kpps to 680 kpps. Within each test run +of the four runs, when number of flows becomes larger, the packet throughput +seems not larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs +between 0 ms and 24 ms with an average leatency of less than 13 ms. The PPS +results are not as consistent as the RTT results, for the mean packet +throughput of the four runs differ from 370 kpps to 582 kpps. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second look similar for the four test runs, whose values change a +lot from 10 pps to 697 kpps. However, the total number of packets received per +second of three runs look similar, which have a large wide range of 2 pps to +1.497 Mpps, while the results on Sep. 18th and 20th have very small maximum +number of packets received per second of 817 kpps. + +In some test runs when running with less than approx. 251000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 251000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on Ericsson POD2_ and LF POD2_ with: +Fuel 9.0 +OpenStack Mitaka +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. diff --git a/docs/results/os-onos-nofeature-ha.rst b/docs/results/os-onos-nofeature-ha.rst new file mode 100644 index 000000000..d8b3ace5f --- /dev/null +++ b/docs/results/os-onos-nofeature-ha.rst @@ -0,0 +1,257 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +====================================== +Test Results for os-onos-nofeature-ha +====================================== + +.. toctree:: + :maxdepth: 2 + + +Joid +===== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD6: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 5 scenario test runs, each run +on the Intel POD6_ between September 13 and 16 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 1.50 and 1.68 ms. +Only one test run has reached greatest RTT spike of 2.62 ms, which has +the smallest RTT of 1.00 ms. The other four runs have no similar spike at all, +the minimum and average RTTs of which are approx. 1.06 ms and 1.32 ms. SLA set +to be 10 ms. The SLA value is used as a reference, it has not been defined by +OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput, which is +measured by fio and the greatest IO read bandwidth of the four runs is 175.4 +MB/s. The IO read bandwidth of the four runs looks similar on different four +days, with an average between 58.1 and 62.0 MB/s, except one on Sep. 14, for +its maximum storage throughput is only 133.0 MB/s. One of the runs has a +minimum BW of 497 KM/s and other has a maximum BW of 177.4 MB/s. The SLA of read +bandwidth sets to be 400 MB/s, which is used as a reference, and it has not +been defined by OPNFV. + +The results of storage IOPS for the five runs look similar with each other. The +IO read times per second of the five test runs have an average value between +1.20 K/s and 1.61 K/s, and meanwhile, the minimum result is only 41 times per +second. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the five runs is between 1.146 ns and 1.172 +ns on average. The variations within each test run are quite different, some +vary from a large range and others have a small change. For example, the +largest change is on September 13, the memory read latency of which is ranging +from 1.152 ns to 1.221 ns. However, the results on September 14 change very +little. The SLA sets to be 30 ns. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC011 +----- +Iperf3 is a tool for evaluating the packet delay variation between 2 VMs on +different blades. The reported packet delay variations of the five test runs +differ from each other. In general, the packet delay of the first two runs look +similar, for they both stay stable within each run. And the mean packet delay of +of them are 0.07714 ms and 0.07982 ms respectively. Of the five runs, the third +has the worst result, because the packet delay reaches 0.08384 ms. The trend of +therest two runs look the same, for the average packet delay are 0.07808 ms and +0.07727 ms respectively. The SLA value sets to be 10 ms. The SLA value is used +as a reference, it has not been defined by OPNFV. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. Among the five test runs, the memory +bandwidth of last three test runs almost keep stable within each run, which is +11.64, 11.71 and 11.61 GB/s on average. However, the memory read and write +bandwidth on Sep. 13 has a large range, for it ranges from 6.68 GB/s to 11.73 +GB/s. Here SLA set to be 15 GB/s. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC014 +----- +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single cpu running and parallel running. It can be seen from the +dashboard that the processing test results vary from scores 3208 to 3314, and +there is only one result one date. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the five test runs is between 259.6 kpps and +318.4 kpps, of which the result of the second run is the highest. The RTT +results of all the test runs keep flat at approx. 20 ms. It is obvious that the +PPS results are not as consistent as the RTT results. + +The No. flows of the five test runs are 240 k on average and the PPS results +look a little waved since the largest packet throughput is 398.9 kpps and the +minimum throughput is 250.6 kpps respectively. + +There are no errors of packets received in the five runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the five +runs have the similar average vaue, that is between 17 ms and 22 ms, of which +the worest RTT is 53 ms on Sep. 14th. + +CPU load is measured by mpstat, and CPU load of the four test runs seem a +little similar, since the minimum value and the peak of CPU load is between 0 +percent and 10 percent respectively. And the best result is obtained on Sep. +13rd, with an CPU load of 10 percent. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test, which +rangs from 21.6 GB/s to 26.8 GB/s and then to 18.4 GB/s on average. Since the +test id is one, it is that only the INT memory write bandwidth is tested. On +the whole, when the block size is 8 kb and 16 kb, the memory write bandwidth +look similar with a minimal BW of 23.0 GB/s and peak value of 28.6 GB/s. And +then with the block size becoming larger, the memory write bandwidth tends to +decrease. SLA sets to be 7 GB/s. The SLA value is used as a a reference, it has +not been defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the five test runs +look similar with each other, and within these test runs, the maximum RTT can +reach 53 ms and the average RTT is usually approx. 18 ms. The network latency +tested on Sep. 14 shows that it has a peak latency of 53 ms. But on the whole, +the average RTTs of the five runs keep flat and the network latency is +relatively short. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 272 MiB on Sep +14. In general, the mean used memory of the five test runs have the similar +trend and the minimum memory used size is approx. 150 MiB, and the average +used memory size is about 250 MiB. On the other hand, for the mean free memory, +the five test runs have the similar trend, whose mean free memory change from +218 MiB to 342 MiB, with an average value of approx. 38 MiB. + +Packet throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +packet throughput of the five test runs seem quite different, ranging from +285.29 kpps to 297.76 kpps. The average number of flows in these tests is +240000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding packet throughput +differ between 250.6k and 398.9k with an average packet throughput between +277.2 K and 318.4 K. In summary, the PPS results seem consistent. Within each +test run of the five runs, when number of flows becomes larger, the packet +throughput seems not larger at the same time. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the five test runs +look similar with each other. Within each test run, the maximum RTT is only 49 +ms and the average RTT is usually approx. 20 ms. On the whole, the average +RTTs of the five runs keep stable and the network latency is relatively short. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool.The largest +cache size is 215 MiB in the four runs, and the smallest cache size is 95 MiB. +On the whole, the average cache size of the five runs change a little and is +about 200 MiB, except the one on Sep. 14th, the mean cache size is very small, +which keeps 102 MiB. Meanwhile, the tread of the buffer size keep flat, since +they have a minimum value of 7 MiB and a maximum value of 8 MiB, with an +average value of about 7.8 MiB. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs seem quite different, ranging from 285.29 kpps to 297.76 +kpps. The average number of flows in these tests is 239.7k, and each run has a +minimum number of flows of 2 and a maximum number of flows of 1.001 Mil. At the +same time, the corresponding packet throughput differ between 227.3k and 398.9k +with an average packet throughput between 277.2k and 318.4k. Within each test +run of the five runs, when number of flows becomes larger, the packet +throughput seems not larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs + between 0 ms and 49 ms with an average leatency of less than 22 ms. The PPS +results are not as consistent as the RTT results, for the mean packet +throughput of the five runs differ from 250.6 kpps to 398.9 kpps. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second look similar for four test runs, whose values change a +lot from 10 pps to 399 kpps, except the one on Sep. 14th, whose total number +of transmitted per second keep stable, that is 10 pps. Similarly, the total +number of packets received per second look the same for four runs, except the +one on Sep. 14th, whose value is only 10 pps. + +In some test runs when running with less than approx. 90000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 250000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on Intel POD6_ with: +Joid +OpenStack Mitaka +Onos Goldeneye +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. diff --git a/docs/results/os-onos-sfc-ha.rst b/docs/results/os-onos-sfc-ha.rst new file mode 100644 index 000000000..e52ae3d55 --- /dev/null +++ b/docs/results/os-onos-sfc-ha.rst @@ -0,0 +1,517 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + + +=============================== +Test Results for os-onos-sfc-ha +=============================== + +.. toctree:: + :maxdepth: 2 + + +fuel +==== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD2: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Ericsson POD2_ or LF POD2_ between September 5 and 10 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 0.5 and 0.6 ms. +A few runs start with a 1 - 1.5 ms RTT spike (This could be because of normal ARP +handling). One test run has a greater RTT spike of 1.9 ms, which is the same +one with the 0.7 ms average. The other runs have no similar spike at all. +To be able to draw conclusions more runs should be made. +SLA set to 10 ms. The SLA value is used as a reference, it has not +been defined by OPNFV. + +TC005 +----- +The IO read bandwidth looks similar between different dates, with an +average between approx. 170 and 200 MB/s. Within each test run the results +vary, with a minimum 2 MB/s and maximum 838 MB/s on the totality. Most runs +have a minimum BW of 3 MB/s (two runs at 2 MB/s). The maximum BW varies more in +absolute numbers between the dates, between 617 and 838 MB/s. +SLA set to 400 MB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC010 +----- +The measurements for memory latency are similar between test dates and result +in approx. 1.2 ns. The variations within each test run are similar, between +1.215 and 1.219 ns. One exception is February 16, where the average is 1.222 +and varies between 1.22 and 1.28 ns. +SLA set to 30 ns. The SLA value is used as a reference, it has not been defined +by OPNFV. + +TC011 +----- +Packet delay variation between 2 VMs on different blades is measured using +Iperf3. On the first date the reported packet delay variation varies between +0.0025 and 0.011 ms, with an average delay variation of 0.0067 ms. +On the second date the delay variation varies between 0.002 and 0.006 ms, with +an average delay variation of 0.004 ms. + +TC012 +----- +Between test dates, the average measurements for memory bandwidth vary between +17.4 and 17.9 GB/s. Within each test run the results vary more, with a minimal +BW of 16.4 GB/s and maximum of 18.2 GB/s on the totality. +SLA set to 15 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC014 +----- +The Unixbench processor test run results vary between scores 3080 and 3240, +one result each date. The average score on the total is 3150. +No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +CPU utilization statistics are collected during UDP flows sent between the VMs +using pktgen as packet generator tool. The average measurements for CPU +utilization ratio vary between 1% to 2%. The peak of CPU utilization ratio +appears around 7%. + +TC069 +----- +Between test dates, the average measurements for memory bandwidth vary between +15.5 and 25.4 GB/s. Within each test run the results vary more, with a minimal +BW of 9.7 GB/s and maximum of 29.5 GB/s on the totality. +SLA set to 6 GB/s. The SLA value is used as a reference, it has not been +defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Memory utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for memory +utilization vary between 225MB to 246MB. The peak of memory utilization appears +around 340MB. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Cache utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The average measurements for cache +utilization vary between 205MB to 212MB. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs at +approx. 15 ms. Some test runs show an increase with many flows, in the range +towards 16 to 17 ms. One exception standing out is Feb. 15 where the average +RTT is stable at approx. 13 ms. The PPS results are not as consistent as the +RTT results. +In some test runs when running with less than approx. 10000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. Around 20 percent decrease in the worst +case. For the other test runs there is however no significant change to the PPS +throughput when the number of flows are increased. In some test runs the PPS +is also greater with 1000000 flows compared to other test runs where the PPS +result is less with only 2 flows. + +The average PPS throughput in the different runs varies between 414000 and +452000 PPS. The total amount of packets in each test run is approx. 7500000 to +8200000 packets. One test run Feb. 15 sticks out with a PPS average of +558000 and approx. 1100000 packets in total (same as the on mentioned earlier +for RTT results). + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally range between 100 and 1000 per test run, +but there are spikes in the range of 10000 lost packets as well, and even +more in a rare cases. + +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. Total number of packets received per +second was average on 200 kpps and total number of packets transmitted per +second was average on 600 kpps. + +Detailed test results +--------------------- +The scenario was run on Ericsson POD2_ and LF POD2_ with: +Fuel 9.0 +OpenStack Mitaka +Onos Goldeneye +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + +Conclusions and recommendations +------------------------------- +The pktgen test configuration has a relatively large base effect on RTT in +TC037 compared to TC002, where there is no background load at all. Approx. +15 ms compared to approx. 0.5 ms, which is more than a 3000 percentage +difference in RTT results. +Especially RTT and throughput come out with better results than for instance +the *fuel-os-nosdn-nofeature-ha* scenario does. The reason for this should +probably be further analyzed and understood. Also of interest could be +to make further analyzes to find patterns and reasons for lost traffic. +Also of interest could be to see if there are continuous variations where +some test cases stand out with better or worse results than the general test +case. + + +Joid +===== + +.. _Grafana: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main +.. _POD6: https://wiki.opnfv.org/pharos?&#community_test_labs + +Overview of test results +------------------------ + +See Grafana_ for viewing test result metrics for each respective test case. It +is possible to chose which specific scenarios to look at, and then to zoom in +on the details of each run test scenario as well. + +All of the test case results below are based on 4 scenario test runs, each run +on the Intel POD6_ between September 8 and 11 in 2016. + +TC002 +----- +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. Most test run measurements result on average between 1.35 ms and 1.57 ms. +Only one test run has reached greatest RTT spike of 2.58 ms. Meanwhile, the +smallest network latency is 1.11 ms, which is obtained on Sep. 11st. In +general, the average of network latency of the four test runs are between 1.35 +ms and 1.57 ms. SLA set to be 10 ms. The SLA value is used as a reference, it +has not been defined by OPNFV. + +TC005 +----- +The IO read bandwidth actually refers to the storage throughput, which is +measured by fio and the greatest IO read bandwidth of the four runs is 175.4 +MB/s. The IO read bandwidth of the three runs looks similar, with an average +between 43.7 and 56.3 MB/s, except one on Sep. 8, for its maximum storage +throughput is only 107.9 MB/s. One of the runs has a minimum BW of 478 KM/s and +other has a maximum BW of 168.6 MB/s. The SLA of read bandwidth sets to be +400 MB/s, which is used as a reference, and it has not been defined by OPNFV. + +The results of storage IOPS for the four runs look similar with each other. The +IO read times per second of the four test runs have an average value between +978 per second and 1.20 K/s, and meanwhile, the minimum result is only 36 times +per second. + +TC010 +----- +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. The memory read latency of the four runs is between 1.164 ns and 1.244 +ns on average. The variations within each test run are quite different, some +vary from a large range and others have a small change. For example, the +largest change is on September 10, the memory read latency of which is ranging +from 1.128 ns to 1.381 ns. However, the results on September 11 change very +little. The SLA sets to be 30 ns. The SLA value is used as a reference, it has +not been defined by OPNFV. + +TC011 +----- +Iperf3 is a tool for evaluating the packet delay variation between 2 VMs on +different blades. The reported packet delay variations of the four test runs +differ from each other. In general, the packet delay of two runs look similar, +for they both stay stable within each run. And the mean packet delay of them +are 0.0772 ms and 0.0788 ms respectively. Of the four runs, the fourth has the +worst result, because the packet delay reaches 0.0838 ms. The rest one has a +large wide range from 0.0666 ms to 0.0798 ms. The SLA value sets to be 10 ms. +The SLA value is used as a reference, it has not been defined by OPNFV. + +TC012 +----- +Lmbench is also used to measure the memory read and write bandwidth, in which +we use bw_mem to obtain the results. Among the four test runs, the trend of the +memory bandwidth almost look similar, which all have a large wide range, and +the minimum and maximum results are 9.02 GB/s and 18.14 GB/s. Here SLA set to +be 15 GB/s. The SLA value is used as a reference, it has not been defined by +OPNFV. + +TC014 +----- +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single cpu running and parallel running. It can be seen from the +dashboard that the processing test results vary from scores 3395 to 3475, and +there is only one result one date. No SLA set. + +TC037 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The mean packet throughput of the four test runs is between 362.1 kpps and +363.5 kpps, of which the result of the third run is the highest. The RTT +results of all the test runs keep flat at approx. 17 ms. It is obvious that the +PPS results are not as consistent as the RTT results. + +The No. flows of the four test runs are 240 k on average and the PPS results +look a little waved since the largest packet throughput is 418.1 kpps and the +minimum throughput is 326.5 kpps respectively. + +There are no errors of packets received in the four runs, but there are still +lost packets in all the test runs. The RTT values obtained by ping of the four +runs have the similar average vaue, that is approx. 17 ms, of which the worst +RTT is 39 ms on Sep. 11st. + +CPU load is measured by mpstat, and CPU load of the four test runs seem a +little similar, since the minimum value and the peak of CPU load is between 0 +percent and nine percent respectively. And the best result is obtained on Sep. +10, with an CPU load of nine percent. + +TC069 +----- +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test, which +rangs from 25.9 GB/s to 26.6 GB/s and then to 18.1 GB/s on average. Since the +test id is one, it is that only the INT memory write bandwidth is tested. On +the whole, when the block size is from 2 kb to 16 kb, the memory write +bandwidth look similar with a minimal BW of 22.1 GB/s and peak value of 28.6 +GB/s. And then with the block size becoming larger, the memory write bandwidth +tends to decrease. SLA sets to be 7 GB/s. The SLA value is used as a reference, +it has not been defined by OPNFV. + +TC070 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other, and within these test runs, the maximum RTT can +reach 39 ms and the average RTT is usually approx. 17 ms. The network latency +tested on Sep. 11 shows that it has a peak latency of 39 ms. But on the whole, +the average RTTs of the five runs keep flat and the network latency is +relatively short. + +Memory utilization is measured by free, which can display amount of free and +used memory in the system. The largest amount of used memory is 270 MiB on the +first two runs. In general, the mean used memory of two test runs have very +large memory utilization, which can reach 264 MiB on average. And the other two +runs have a large wide range of memory usage with the minimum value of 150 MiB +and the maximum value of 270 MiB. On the other hand, for the mean free memory, +the four test runs have the similar trend with that of the mean used memory. +In general, the mean free memory change from 220 MiB to 342 MiB. + +Packet throughput and packet loss can be measured by pktgen, which is a tool +in the network for generating traffic loads for network experiments. The mean +packet throughput of the four test runs seem quite different, ranging from +326.5 kpps to 418.1 kpps. The average number of flows in these tests is +240000, and each run has a minimum number of flows of 2 and a maximum number +of flows of 1.001 Mil. At the same time, the corresponding packet throughput +differ between 326.5 kpps and 418.1 kpps with an average packet throughput between +361.7 kpps and 363.5 kpps. In summary, the PPS results seem consistent. Within each +test run of the four runs, when number of flows becomes larger, the packet +throughput seems not larger at the same time. + +TC071 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The network latency is measured by ping, and the results of the four test runs +look similar with each other. Within each test run, the maximum RTT is only 47 +ms and the average RTT is usually approx. 15 ms. On the whole, the average +RTTs of the four runs keep stable and the network latency is relatively small. + +Cache utilization is measured by cachestat, which can display size of cache and +buffer in the system. Cache utilization statistics are collected during UDP +flows sent between the VMs using pktgen as packet generator tool. The largest +cache size is 214 MiB, which is same for the four runs, and the smallest cache +size is 94 MiB. On the whole, the average cache size of the four runs look the +same and is between 198 MiB and 207 MiB. Meanwhile, the tread of the buffer +size keep flat, since they have a minimum value of 7 MiB and a maximum value of +8 MiB, with an average value of about 7.9 MiB. + +Packet throughput can be measured by pktgen, which is a tool in the network for +generating traffic loads for network experiments. The mean packet throughput of +the four test runs seem quite the same, which is approx. 363 kpps. The average +number of flows in these tests is 240k, and each run has a minimum number of +flows of 2 and a maximum number of flows of 1.001 Mil. At the same time, the +corresponding packet throughput differ between 327 kpps and 418 kpps with an +average packet throughput of about 363 kpps. Within each test run of the four +runs, when number of flows becomes larger, the packet throughput seems not +larger in the meantime. + +TC072 +----- +The amount of packets per second (PPS) and round trip times (RTT) between 2 VMs +on different blades are measured when increasing the amount of UDP flows sent +between the VMs using pktgen as packet generator tool. + +Round trip times and packet throughput between VMs can typically be affected by +the amount of flows set up and result in higher RTT and less PPS throughput. + +The RTT results are similar throughout the different test dates and runs +between 0 ms and 47 ms with an average leatency of less than 16 ms. The PPS +results are not as consistent as the RTT results, for the mean packet +throughput of the four runs differ from 361.7 kpps to 365.0 kpps. + +Network utilization is measured by sar, that is system activity reporter, which +can display the average statistics for the time since the system was started. +Network utilization statistics are collected during UDP flows sent between the +VMs using pktgen as packet generator tool. The largest total number of packets +transmitted per second look similar for two test runs, whose values change a +lot from 10 pps to 432 kpps. While results of the other test runs seem the same +and keep stable with the average number of packets transmitted per second of 10 +pps. However, the total number of packets received per second of the four runs +look similar, which have a large wide range of 2 pps to 657 kpps. + +In some test runs when running with less than approx. 250000 flows the PPS +throughput is normally flatter compared to when running with more flows, after +which the PPS throughput decreases. For the other test runs there is however no +significant change to the PPS throughput when the number of flows are +increased. In some test runs the PPS is also greater with 250000 flows +compared to other test runs where the PPS result is less with only 2 flows. + +There are lost packets reported in most of the test runs. There is no observed +correlation between the amount of flows and the amount of lost packets. +The lost amount of packets normally differs a lot per test run. + +Detailed test results +--------------------- +The scenario was run on Intel POD6_ with: +Joid +OpenStack Mitaka +Onos Goldeneye +OpenVirtualSwitch 2.5.90 +OpenDayLight Beryllium + +Rationale for decisions +----------------------- +Pass + +Conclusions and recommendations +------------------------------- +Tests were successfully executed and metrics collected. +No SLA was verified. To be decided on in next release of OPNFV. + diff --git a/docs/results/overview.rst b/docs/results/overview.rst index 7f3a34e56..b4a050545 100644 --- a/docs/results/overview.rst +++ b/docs/results/overview.rst @@ -3,24 +3,10 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Ericsson AB and others. -===================== -Yardstick Test Report -===================== - -.. toctree:: - :maxdepth: 2 - -Introduction -============ - -Document Identifier -------------------- - -This document is part of deliverables of the OPNFV release brahmaputra.3.0 - -Scope ------ +Yardstick test tesult document overview +======================================= +.. _`Yardstick user guide`: artifacts.opnfv.org/yardstick/docs/userguide/index.html .. _Dashboard: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main .. _Jenkins: https://build.opnfv.org/ci/view/yardstick/ .. _Scenarios: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-scenarios @@ -28,70 +14,93 @@ Scope This document provides an overview of the results of test cases developed by the OPNFV Yardstick Project, executed on OPNFV community labs. -OPNFV Continous Integration provides automated build, deploy and testing for -the software developed in OPNFV. Unless stated, the reported tests are -automated via Jenkins Jobs. - -Test results are visible in the following dashboard: - -* *Yardstick* Dashboard_: uses influx DB to store test results and Grafana for - visualization (user: opnfv/ password: opnfv) - - -References ----------- - -* IEEE Std 829-2008. "Standard for Software and System Test Documentation". - -* OPNFV Brahamputra release note for Yardstick. - - - -General -======= +Yardstick project is described in `Yardstick user guide`_. -Yardstick Test Cases have been executed for scenarios and features defined in -this OPNFV release. +Yardstick is run systematically at the end of an OPNFV fresh installation. +The system under test (SUT) is installed by the installer Apex, Compass, Fuel +or Joid on Performance Optimized Datacenter (POD); One single installer per +POD. All the runnable test cases are run sequentially. The installer and the +POD are considered to evaluate whether the test case can be run or not. That is +why all the number of test cases may vary from 1 installer to another and from +1 POD to POD. -The test environments were installed by one of the following: Apex, Compass, -Fuel or Joid; one single installer per POD. +OPNFV CI provides automated build, deploy and testing for +the software developed in OPNFV. Unless stated, the reported tests are +automated via Jenkins Jobs. Yardsrick test results from OPNFV Continous +Integration can be found in the following dashboard: -The results of executed tests are available in Dashboard_ and all logs stored -in Jenkins_. +* *Yardstick* Dashboard_: uses influx DB to store Yardstick CI test results and + Grafana for visualization (user: opnfv/ password: opnfv) -After one week of measurments, in general, SDN ONOS showed lower latency than -SDN ODL, which showed lower latency than an environment installed with pure -OpenStack. Additional time and PODs make this a non-conclusive statement, see -Scenarios_ for a snapshot and Dashboard_ for complete results. +The results of executed test cases are available in Dashboard_ and all logs are +stored in Jenkins_. It was not possible to execute the entire Yardstick test cases suite on the PODs assigned for release verification over a longer period of time, due to continuous work on the software components and blocking faults either on -environment, feature or test framework. - -Four consecutive successful runs was defined as criteria for release. -It is a recommendation to run Yardstick test cases over a longer period -of time in order to better understand the behavior of the system. - +environment, features or test framework. + +The list of scenarios supported by each installer can be described as follows: + ++-------------------------+---------+---------+---------+---------+ +| Scenario | Apex | Compass | Fuel | Joid | ++=========================+=========+=========+=========+=========+ +| os-nosdn-nofeature-noha | | | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-nofeature-ha | X | X | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-nofeature-ha | X | X | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-nofeature-noha| | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l3-nofeature-ha | X | X | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l3-nofeature-noha| | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-onos-sfc-ha | X | X | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-onos-sfc-noha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-onos-nofeature-ha | X | X | X | X | ++-------------------------+---------+---------+---------+---------+ +| os-onos-nofeature-noha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-sfc-ha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-sfc-noha | X | X | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-bgpvpn-ha | X | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-bgpvpn-noha | | X | X | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-kvm-ha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-kvm-noha | | X | X | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-ovs-ha | | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-ovs-noha | X | | X | | ++-------------------------+---------+---------+---------+---------+ +| os-ocl-nofeature-ha | | | | | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-lxd-ha | | | | X | ++-------------------------+---------+---------+---------+---------+ +| os-nosdn-lxd-noha | | | | X | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-fdio-noha | X | | | | ++-------------------------+---------+---------+---------+---------+ +| os-odl_l2-moon-ha | | X | | | ++-------------------------+---------+---------+---------+---------+ + +To qualify for release, the scenarios must have deployed and been successfully +tested in four consecutive installations to establish stability of deployment +and feature capability. It is a recommendation to run Yardstick test +cases over a longer period of time in order to better understand the behavior +of the system under test. +References +---------- -Document change procedures and history --------------------------------------- +* IEEE Std 829-2008. "Standard for Software and System Test Documentation". -+--------------------------------------+--------------------------------------+ -| **Project** | Yardstick | -| | | -+--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.3.0 | -| | | -+--------------------------------------+--------------------------------------+ -| **Release designation** | Brahmaputra | -| | | -+--------------------------------------+--------------------------------------+ -| **Release date** | Apr 27th, 2016 | -| | | -+--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | OPNFV Brahmaputra release test | -| | results. | -| | | -+--------------------------------------+--------------------------------------+ +* OPNFV Colorado release note for Yardstick. diff --git a/docs/results/results.rst b/docs/results/results.rst index f3831b865..04c6b9f87 100644 --- a/docs/results/results.rst +++ b/docs/results/results.rst @@ -2,14 +2,13 @@ .. License. .. http://creativecommons.org/licenses/by/4.0 +Results listed by scenario +========================== -====================== -Yardstick Test Results -====================== - -.. toctree:: - :maxdepth: 2 - +The following sections describe the yardstick results as evaluated for the +Colorado release scenario validation runs. Each section describes the +determined state of the specific scenario as deployed in the Colorado +release process. Scenario Results ================ @@ -21,61 +20,21 @@ The following documents contain results of Yardstick test cases executed on OPNFV labs, triggered by OPNFV CI pipeline, documented per scenario. -Ready scenarios ---------------- - -The following scenarios run at least four consecutive times Yardstick test -cases suite: - .. toctree:: :maxdepth: 1 - apex-os-odl_l2-nofeature-ha.rst - compass-os-nosdn-nofeature-ha.rst - compass-os-odl_l2-nofeature-ha.rst - compass-os-onos-nofeature-ha.rst - fuel-os-nosdn-nofeature-ha.rst - fuel-os-odl_l2-nofeature-ha.rst - fuel-os-onos-nofeature-ha.rst - fuel-os-nosdn-kvm-ha - joid-os-odl_l2-nofeature-ha.rst - - -Limitations ------------ - -For the following scenarios, Yardstick generic test cases suite was executed at -least one time however less than four consecutive times, measurements -collected: - - - * fuel-os-odl_l2-bgpvpn-ha - - * fuel-os-odl_l3-nofeature-ha - - * joid-os-nosdn-nofeature-ha - - * joid-os-onos-nofeature-ha - - -For the following scenario, Yardstick generic test cases suite was executed -four consecutive times, measurements collected; no feature test cases were -executed, therefore the feature is not verified by Yardstick: - - * apex-os-odl_l2-bgpvpn-ha - - -For the following scenario, Yardstick generic test cases suite was executed -three consecutive times, measurements collected; no feature test cases -were executed, therefore the feature is not verified by Yardstick: - - * fuel-os-odl_l2-sfc-ha - + os-nosdn-nofeature-ha.rst + os-nosdn-nofeature-noha.rst + os-odl_l2-nofeature-ha.rst + os-odl_l2-bgpvpn-ha.rst + os-odl_l2-sfc-ha.rst + os-nosdn-kvm-ha.rst + os-onos-nofeature-ha.rst + os-onos-sfc-ha.rst Test results of executed tests are avilable in Dashboard_ and logs in Jenkins_. - Feature Test Results ==================== @@ -91,5 +50,8 @@ The following features were verified by Yardstick test cases: * Virtual Traffic Classifier (see :doc:`yardstick-opnfv-vtc`) + * StorPerf + .. note:: The test cases for IPv6 and Parser Projects are included in the compass scenario. + diff --git a/docs/results/yardstick-opnfv-ha.rst b/docs/results/yardstick-opnfv-ha.rst index 4ee9de847..ef1617342 100644 --- a/docs/results/yardstick-opnfv-ha.rst +++ b/docs/results/yardstick-opnfv-ha.rst @@ -114,5 +114,5 @@ There are several improvement points for HA test: a) Running test cases in different enveriment deployed by different installers, such as compass4nfv, apex and joid, with different versiones. b) The period of each request is a little long, it needs more accurate test - method. +method. c) More test cases with different faults and different monitors are needed. diff --git a/docs/templates/Yardstick_task_templates.rst b/docs/templates/Yardstick_task_templates.rst index 8185062b2..c8b6f6e77 100755 --- a/docs/templates/Yardstick_task_templates.rst +++ b/docs/templates/Yardstick_task_templates.rst @@ -77,7 +77,7 @@ a JSON or YAML dictionary): :: yardstick task start samples/ping-template.yaml - --task-args'{"packetsize":"200"}' + --task-args '{"packetsize":"200"}' 2.Refer to a file that specifies the argument values (JSON/YAML): diff --git a/docs/userguide/01-introduction.rst b/docs/userguide/01-introduction.rst index 3db6ce001..9d9cf0fb5 100755 --- a/docs/userguide/01-introduction.rst +++ b/docs/userguide/01-introduction.rst @@ -40,18 +40,25 @@ This document consists of the following chapters: * Chapter :doc:`02-methodology` describes the methodology implemented by the Yardstick Project for :term:`NFVI` verification. -* Chapter :doc:`architecture` provides information on the software architecture +* Chapter :doc:`03-architecture` provides information on the software architecture of yardstick. + * Chapter :doc:`04-vtc-overview` provides information on the :term:`VTC`. -* Chapter :doc:`apexlake_installation` provides instructions to install the - experimental framework *ApexLake* and chapter :doc:`apexlake_api` explains +* Chapter :doc:`05-apexlake_installation` provides instructions to install the + experimental framework *ApexLake* and chapter :doc:`06-apexlake_api` explains how this framework is integrated in *Yardstick*. -* Chapter :doc:`03-installation` provides instructions to install *Yardstick*. +* Chapter :doc:`07-installation` provides instructions to install *Yardstick*. + +* Chapter :doc:`08-yardstick_plugin` provides information on how to integrate + other OPNFV testing projects into *Yardstick*. -* Chapter :doc:`03-list-of-tcs` includes a list of available Yardstick - test cases. +* Chapter :doc:`09-result-store-InfluxDB` provides inforamtion on how to run + plug-in test cases and store test results into community's InfluxDB. + +* Chapter :doc:`10-list-of-tcs` includes a list of available Yardstick test + cases. Contact Yardstick @@ -60,3 +67,4 @@ Contact Yardstick Feedback? `Contact us`_ .. _Contact us: opnfv-users@lists.opnfv.org + diff --git a/docs/userguide/02-methodology.rst b/docs/userguide/02-methodology.rst index 3fa432a98..34d271095 100644 --- a/docs/userguide/02-methodology.rst +++ b/docs/userguide/02-methodology.rst @@ -59,7 +59,7 @@ The metrics, as defined by ETSI GS NFV-TST001, are shown in :ref:`Table1 <table2_1>`, :ref:`Table2 <table2_2>` and :ref:`Table3 <table2_3>`. -In OPNFV Brahmaputra release, generic test cases covering aspects of the listed +In OPNFV Colorado release, generic test cases covering aspects of the listed metrics are available; further OPNFV releases will provide extended testing of these metrics. The view of available Yardstick test cases cross ETSI definitions in @@ -169,24 +169,27 @@ options). | | | | | +---------+-------------------+----------------+------------------------------+ | Compute | TC003 [1]_ | TC003 [1]_ | TC013 [1]_ | -| | TC004 [1]_ | TC004 [1]_ | TC015 [1]_ | -| | TC014 | TC010 | | -| | TC024 | TC012 | | -| | | | | +| | TC004 | TC004 | TC015 [1]_ | +| | TC010 | TC024 | | +| | TC012 | TC055 | | +| | TC014 | | | +| | TC069 | | | +---------+-------------------+----------------+------------------------------+ -| Network | TC002 | TC001 | TC016 [1]_ | -| | TC011 | TC008 | TC018 [1]_ | -| | | TC009 | | -| | | | | +| Network | TC001 | TC044 | TC016 [1]_ | +| | TC002 | TC073 | TC018 [1]_ | +| | TC009 | TC075 | | +| | TC011 | | | +| | TC042 | | | +| | TC043 | | | +---------+-------------------+----------------+------------------------------+ -| Storage | TC005 | TC005 | TC017 [1]_ | -| | | | | +| Storage | TC005 | TC063 | TC017 [1]_ | +---------+-------------------+----------------+------------------------------+ .. note:: The description in this OPNFV document is intended as a reference for users to understand the scope of the Yardstick Project and the deliverables of the Yardstick framework. For complete description of - the methodology, refer to the ETSI document. + the methodology, please refer to the ETSI document. .. rubric:: Footnotes .. [1] To be included in future deliveries. + diff --git a/docs/userguide/03-architecture.rst b/docs/userguide/03-architecture.rst index 3abb67b7d..ace3117c2 100755 --- a/docs/userguide/03-architecture.rst +++ b/docs/userguide/03-architecture.rst @@ -222,7 +222,7 @@ Deployment View =============== Yardstick deployment view shows how the yardstick tool can be deployed into the underlying platform. Generally, yardstick tool is installed on JumpServer(see -`03-installation` for detail installation steps), and JumpServer is +`07-installation` for detail installation steps), and JumpServer is connected with other control/compute servers by networking. Based on this deployment, yardstick can run the test cases on these hosts, and get the test result for better showing. @@ -256,8 +256,11 @@ Yardstick Directory structure by Heat. Currently contains how to build the yardstick-trusty-server image with the different tools that are needed from within the image. +*plugin/* - Plug-in configuration files are stored here. + *vTC/* - Contains the files for running the virtual Traffic Classifier tests. *yardstick/* - Contains the internals of Yardstick: Runners, Scenario, Contexts, - CLI parsing, keys, plotting tools, dispatcher and so on. + CLI parsing, keys, plotting tools, dispatcher, plugin + install/remove scripts and so on. diff --git a/docs/userguide/07-installation.rst b/docs/userguide/07-installation.rst index 475719c72..9c2082a27 100644 --- a/docs/userguide/07-installation.rst +++ b/docs/userguide/07-installation.rst @@ -9,19 +9,52 @@ Yardstick Installation Abstract -------- -Yardstick currently supports installation on Ubuntu 14.04 or by using a Docker -image. Detailed steps about installing Yardstick using both of these options -can be found below. +Yardstick supports installation on Ubuntu 14.04 or via a Docker image. The +installation procedure on Ubuntu 14.04 or via the docker image are detailed in +the section below. -To use Yardstick you should have access to an OpenStack environment, -with at least Nova, Neutron, Glance, Keystone and Heat installed. +To use Yardstick you should have access to an OpenStack environment, with at +least Nova, Neutron, Glance, Keystone and Heat installed. The steps needed to run Yardstick are: -1. Install Yardstick and create the test configuration .yaml file. -2. Build a guest image and load the image into the OpenStack environment. -3. Create a Neutron external network and load OpenStack environment variables. -4. Run the test case. +1. Install Yardstick. +2. Load OpenStack environment variables. +3. Create a Neutron external network. +4. Build Yardstick flavor and a guest image. +5. Load the guest image into the OpenStack environment. +6. Create the test configuration .yaml file. +7. Run the test case. + + +Prerequisites +------------- + +The OPNFV deployment is out of the scope of this document but it can be +found in http://artifacts.opnfv.org/opnfvdocs/colorado/docs/configguide/index.html. +The OPNFV platform is considered as the System Under Test (SUT) in this +document. + +Several prerequisites are needed for Yardstick: + + #. A Jumphost to run Yardstick on + #. A Docker daemon shall be installed on the Jumphost + #. A public/external network created on the SUT + #. Connectivity from the Jumphost to the SUT public/external network + +WARNING: Connectivity from Jumphost is essential and it is of paramount +importance to make sure it is working before even considering to install +and run Yardstick. Make also sure you understand how your networking is +designed to work. + +NOTE: **Jumphost** refers to any server which meets the previous +requirements. Normally it is the same server from where the OPNFV +deployment has been triggered previously. + +NOTE: If your Jumphost is operating behind a company http proxy and/or +Firewall, please consult first the section `Proxy Support`_, towards +the end of this document. The section details some tips/tricks which +*may* be of help in a proxified environment. Installing Yardstick on Ubuntu 14.04 @@ -29,195 +62,177 @@ Installing Yardstick on Ubuntu 14.04 .. _install-framework: -Installing Yardstick framework -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Install dependencies: -:: +You can install Yardstick framework directly on Ubuntu 14.04 or in an Ubuntu +14.04 Docker image. No matter which way you choose to install Yardstick +framework, the following installation steps are identical. + +If you choose to use the Ubuntu 14.04 Docker image, You can pull the Ubuntu +14.04 Docker image from Docker hub: - sudo apt-get update && sudo apt-get install -y \ - wget \ - git \ - sshpass \ - qemu-utils \ - kpartx \ - libffi-dev \ - libssl-dev \ - python \ - python-dev \ - python-virtualenv \ - libxml2-dev \ - libxslt1-dev \ - python-setuptools - -Create a python virtual environment, source it and update setuptools: :: - virtualenv ~/yardstick_venv - source ~/yardstick_venv/bin/activate - easy_install -U setuptools + docker pull ubuntu:14.04 +Installing Yardstick framework +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Download source code and install python dependencies: + :: git clone https://gerrit.opnfv.org/gerrit/yardstick cd yardstick - python setup.py install + ./install.sh -There is also a YouTube video, showing the above steps: -.. image:: http://img.youtube.com/vi/4S4izNolmR0/0.jpg - :alt: http://www.youtube.com/watch?v=4S4izNolmR0 - :target: http://www.youtube.com/watch?v=4S4izNolmR0 +Installing Yardstick using Docker +--------------------------------- -Installing extra tools -^^^^^^^^^^^^^^^^^^^^^^ -yardstick-plot -"""""""""""""" -Yardstick has an internal plotting tool ``yardstick-plot``, which can be installed -using the following command: -:: +Yardstick has a Docker image, this Docker image (**Yardstick-stable**) +serves as a replacement for installing the Yardstick framework in a virtual +environment (for example as done in :ref:`install-framework`). +It is recommended to use this Docker image to run Yardstick test. - sudo apt-get install -y g++ libfreetype6-dev libpng-dev pkg-config - python setup.py develop easy_install yardstick[plot] +Pulling the Yardstick Docker image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. _guest-image: +.. _dockerhub: https://hub.docker.com/r/opnfv/yardstick/ -Building a guest image -^^^^^^^^^^^^^^^^^^^^^^ -Yardstick has a tool for building an Ubuntu Cloud Server image containing all -the required tools to run test cases supported by Yardstick. It is necessary to -have sudo rights to use this tool. +Pull the Yardstick Docker image ('opnfv/yardstick') from the public dockerhub +registry under the OPNFV account: [dockerhub_], with the following docker +command:: -Also you may need install several additional packages to use this tool, by -follwing the commands below: -:: + docker pull opnfv/yardstick:stable - apt-get update && apt-get install -y \ - qemu-utils \ - kpartx +After pulling the Docker image, check that it is available with the +following docker command:: + + [yardsticker@jumphost ~]$ docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + opnfv/yardstick stable a4501714757a 1 day ago 915.4 MB + +Run the Docker image: -This image can be built using the following command while in the directory where -Yardstick is installed (``~/yardstick`` if the framework is installed -by following the commands above): :: - sudo ./tools/yardstick-img-modify tools/ubuntu-server-cloudimg-modify.sh + docker run --privileged=true -it opnfv/yardstick:stable /bin/bash -**Warning:** the script will create files by default in: -``/tmp/workspace/yardstick`` and the files will be owned by root! +In the container the Yardstick repository is located in the /home/opnfv/repos +directory. -The created image can be added to OpenStack using the ``glance image-create`` or -via the OpenStack Dashboard. -Example command: -:: +OpenStack parameters and credentials +------------------------------------ - glance --os-image-api-version 1 image-create \ - --name yardstick-trusty-server --is-public true \ - --disk-format qcow2 --container-format bare \ - --file /tmp/workspace/yardstick/yardstick-trusty-server.img +Environment variables +^^^^^^^^^^^^^^^^^^^^^ +Before running Yardstick it is necessary to export OpenStack environment variables +from the OpenStack *openrc* file (using the ``source`` command) and export the +external network name ``export EXTERNAL_NETWORK="external-network-name"``, +the default name for the external network is ``net04_ext``. +Credential environment variables in the *openrc* file have to include at least: -Installing Yardstick using Docker +* OS_AUTH_URL +* OS_USERNAME +* OS_PASSWORD +* OS_TENANT_NAME + +A sample openrc file may look like this: + +* export OS_PASSWORD=console +* export OS_TENANT_NAME=admin +* export OS_AUTH_URL=http://172.16.1.222:35357/v2.0 +* export OS_USERNAME=admin +* export OS_VOLUME_API_VERSION=2 +* export EXTERNAL_NETWORK=net04_ext + + +Yardstick falvor and guest images --------------------------------- -Yardstick has two Docker images, first one (**Yardstick-framework**) serves as a -replacement for installing the Yardstick framework in a virtual environment (for -example as done in :ref:`install-framework`), while the other image is mostly for -CI purposes (**Yardstick-CI**). +Before executing Yardstick test cases, make sure that yardstick guest image and +yardstick flavor are available in OpenStack. +Detailed steps about creating yardstick flavor and building yardstick-trusty-server +image can be found below. -Yardstick-framework image -^^^^^^^^^^^^^^^^^^^^^^^^^ -Download the source code: +Yardstick-flavor +^^^^^^^^^^^^^^^^ +Most of the sample test cases in Yardstick are using an OpenStack flavor called +*yardstick-flavor* which deviates from the OpenStack standard m1.tiny flavor by the +disk size - instead of 1GB it has 3GB. Other parameters are the same as in m1.tiny. + +Create yardstick-flavor: :: - git clone https://gerrit.opnfv.org/gerrit/yardstick + nova flavor-create yardstick-flavor 100 512 3 1 -Build the Docker image and tag it as *yardstick-framework*: -:: +.. _guest-image: - cd yardstick - docker build -t yardstick-framework . +Building a guest image +^^^^^^^^^^^^^^^^^^^^^^ +Most of the sample test cases in Yardstick are using a guest image called +*yardstick-trusty-server* which deviates from an Ubuntu Cloud Server image +containing all the required tools to run test cases supported by Yardstick. +Yardstick has a tool for building this custom image. It is necessary to have +sudo rights to use this tool. -Run the Docker instance: +Also you may need install several additional packages to use this tool, by +follwing the commands below: :: - docker run --name yardstick_instance -i -t yardstick-framework - -To build a guest image for Yardstick, see :ref:`guest-image`. + apt-get update && apt-get install -y \ + qemu-utils \ + kpartx -Yardstick-CI image -^^^^^^^^^^^^^^^^^^ -Pull the Yardstick-CI Docker image from Docker hub: +This image can be built using the following command while in the directory where +Yardstick is installed (``~/yardstick`` if the framework is installed +by following the commands above): :: - docker pull opnfv/yardstick:$DOCKER_TAG + export YARD_IMG_ARCH="amd64" + sudo echo "Defaults env_keep += \"YARD_IMG_ARCH\"" >> /etc/sudoers + sudo ./tools/yardstick-img-modify tools/ubuntu-server-cloudimg-modify.sh -Where ``$DOCKER_TAG`` is latest for master branch, as for the release branches, -this coincides with its release name, such as brahmaputra.1.0. +**Warning:** the script will create files by default in: +``/tmp/workspace/yardstick`` and the files will be owned by root! -Run the Docker image: +If you are building this guest image in inside a docker container make sure the +container is granted with privilege. + +The created image can be added to OpenStack using the ``glance image-create`` or +via the OpenStack Dashboard. + +Example command: :: - docker run \ - --privileged=true \ - --rm \ - -t \ - -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ - -e "INSTALLER_IP=${INSTALLER_IP}" \ - opnfv/yardstick \ - exec_tests.sh ${YARDSTICK_DB_BACKEND} ${YARDSTICK_SUITE_NAME} - -Where ``${INSTALLER_TYPE}`` can be apex, compass, fuel or joid, ``${INSTALLER_IP}`` -is the installer master node IP address (i.e. 10.20.0.2 is default for fuel). ``${YARDSTICK_DB_BACKEND}`` -is the IP and port number of DB, ``${YARDSTICK_SUITE_NAME}`` is the test suite you want to run. -For more details, please refer to the Jenkins job defined in Releng project, labconfig information -and sshkey are required. See the link -https://git.opnfv.org/cgit/releng/tree/jjb/yardstick/yardstick-ci-jobs.yml. - -Note: exec_tests.sh is used for executing test suite here, furthermore, if someone wants to execute the -test suite manually, it can be used as long as the parameters are configured correct. Another script -called run_tests.sh is used for unittest in Jenkins verify job, in local manaul environment, -it is recommended to run before test suite execuation. - -Basic steps performed by the **Yardstick-CI** container: - -1. clone yardstick and releng repos -2. setup OS credentials (releng scripts) -3. install yardstick and dependencies -4. build yardstick cloud image and upload it to glance -5. upload cirros-0.3.3 cloud image to glance -6. run yardstick test scenarios -7. cleanup + glance --os-image-api-version 1 image-create \ + --name yardstick-image --is-public true \ + --disk-format qcow2 --container-format bare \ + --file /tmp/workspace/yardstick/yardstick-image.img +Some Yardstick test cases use a Cirros image, you can find one at +http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img -OpenStack parameters and credentials ------------------------------------- -Yardstick-flavor -^^^^^^^^^^^^^^^^ -Most of the sample test cases in Yardstick are using an OpenStack flavor called -*yardstick-flavor* which deviates from the OpenStack standard m1.tiny flavor by the -disk size - instead of 1GB it has 3GB. Other parameters are the same as in m1.tiny. +Automatic flavor and image creation +----------------------------------- +Yardstick has a script for automatic creating yardstick flavor and building +guest images. This script is mainly used in CI, but you can still use it in +your local environment. -Environment variables -^^^^^^^^^^^^^^^^^^^^^ -Before running Yardstick it is necessary to export OpenStack environment variables -from the OpenStack *openrc* file (using the ``source`` command) and export the -external network name ``export EXTERNAL_NETWORK="external-network-name"``, -the default name for the external network is ``net04_ext``. +Example command: -Credential environment variables in the *openrc* file have to include at least: +:: + + export YARD_IMG_ARCH="amd64" + sudo echo "Defaults env_keep += \"YARD_IMG_ARCH\"" >> /etc/sudoers + source $YARDSTICK_REPO_DIR/tests/ci/load_images.sh -* OS_AUTH_URL -* OS_USERNAME -* OS_PASSWORD -* OS_TENANT_NAME Yardstick default key pair ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -230,8 +245,10 @@ Examples and verifying the install ---------------------------------- It is recommended to verify that Yardstick was installed successfully -by executing some simple commands and test samples. Below is an example invocation -of yardstick help command and ping.py test sample: +by executing some simple commands and test samples. Before executing yardstick +test cases make sure yardstick flavor and building yardstick-trusty-server +image can be found in glance and openrc file is sourced. Below is an example +invocation of yardstick help command and ping.py test sample: :: yardstick –h @@ -240,18 +257,8 @@ of yardstick help command and ping.py test sample: Each testing tool supported by Yardstick has a sample configuration file. These configuration files can be found in the **samples** directory. -Example invocation of ``yardstick-plot`` tool: -:: - - yardstick-plot -i /tmp/yardstick.out -o /tmp/plots/ - Default location for the output is ``/tmp/yardstick.out``. -More info about the tool can be found by executing: -:: - - yardstick-plot -h - Deploy InfluxDB and Grafana locally ------------------------------------ @@ -259,6 +266,7 @@ Deploy InfluxDB and Grafana locally .. pull docker images Pull docker images + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: @@ -299,6 +307,8 @@ Config grafana log on using admin/admin and config database resource to be {YOUR_IP_HERE}:8086 .. image:: images/Grafana_config.png + :width: 800px + :alt: Grafana data source configration Config yardstick conf ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -319,7 +329,7 @@ Config yardstick.conf username = root password = root -Now you can run yardstick test case and store the results in influxdb +Now you can run yardstick test cases and store the results in influxdb ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -350,12 +360,12 @@ fuel_test_suite.yaml - file_name: iperf3.yaml -As you can see, there are two test cases in fuel_test_suite, the syntas is simple +As you can see, there are two test cases in fuel_test_suite, the syntax is simple here, you must specify the schema and the name, then you just need to list the test cases in the tag "test_cases" and also mark their relative directory in the tag "test_cases_dir". -Yardstick test suite also support constraints and task args for each test suite. +Yardstick test suite also support constraints and task args for each test case. Here is another sample to show this, which is digested from one big test suite. os-nosdn-nofeature-ha.yaml @@ -382,9 +392,10 @@ os-nosdn-nofeature-ha.yaml huawei-pod1: '{"pod_info": "etc/yardstick/.../pod.yaml", "host": "node4.LF","target": "node5.LF"}' -As you can see in test case "opnfv_yardstick_tc043.yaml", it has two tags, "constraint" and +As you can see in test case "opnfv_yardstick_tc043.yaml", there are two tags, "constraint" and "task_args". "constraint" is where you can specify which installer or pod it can be run in the ci environment. "task_args" is where you can specify the task arguments for each pod. All in all, to create a test suite in yardstick, you just need to create a suite yaml file and add test cases and constraint or task arguments if necessary. + diff --git a/docs/userguide/08-yardstick_plugin.rst b/docs/userguide/08-yardstick_plugin.rst index e68db650d..f16dedd02 100644 --- a/docs/userguide/08-yardstick_plugin.rst +++ b/docs/userguide/08-yardstick_plugin.rst @@ -48,11 +48,11 @@ environment and other dependencies: 3. Make sure Jump Host have access to the OpenStack Controller API. 4. Make sure Jump Host must have internet connectivity for downloading docker image. 5. You need to know where to get basic openstack Keystone authorization info, such as -OS_PASSWORD, OS_TENANT_NAME, OS_AUTH_URL, OS_USERNAME. + OS_PASSWORD, OS_TENANT_NAME, OS_AUTH_URL, OS_USERNAME. 6. To run a Storperf container, you need to have OpenStack Controller environment -variables defined and passed to Storperf container. The best way to do this is to -put environment variables in a "storperf_admin-rc" file. The storperf_admin-rc -should include credential environment variables at least: + variables defined and passed to Storperf container. The best way to do this is to + put environment variables in a "storperf_admin-rc" file. The storperf_admin-rc + should include credential environment variables at least: * OS_AUTH_URL * OS_TENANT_ID diff --git a/docs/userguide/09-result-store-InfluxDB.rst b/docs/userguide/09-result-store-InfluxDB.rst index 5c49e9f7c..a0bb48a80 100644 --- a/docs/userguide/09-result-store-InfluxDB.rst +++ b/docs/userguide/09-result-store-InfluxDB.rst @@ -17,7 +17,7 @@ into community's InfluxDB. The framework is shown in Framework_. .. image:: images/InfluxDB_store.png - :width: 1200px + :width: 800px :alt: Store Other Project's Test Results in InfluxDB Store Storperf Test Results into Community's InfluxDB @@ -81,6 +81,6 @@ can be accessed by Login_. .. image:: images/results_visualization.png - :width: 1200px + :width: 800px :alt: results visualization diff --git a/docs/userguide/10-grafana.rst b/docs/userguide/10-grafana.rst new file mode 100644 index 000000000..416857b71 --- /dev/null +++ b/docs/userguide/10-grafana.rst @@ -0,0 +1,119 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2016 Huawei Technologies Co.,Ltd and others + +================= +Grafana dashboard +================= + + +Abstract +======== + +This chapter describes the Yardstick grafana dashboard. The Yardstick grafana +dashboard can be found here: http://testresults.opnfv.org/grafana/ + + +.. image:: images/login.png + :width: 800px + :alt: Yardstick grafana dashboard + + +Public access +============= + +Yardstick provids a public account for accessing to the dashboard. The username +and password are both set to ‘opnfv’. + + +Testcase dashboard +================== + +For each test case, there is a dedicated dashboard. Shown here is the dashboard +of TC002. + + +.. image:: images/TC002.png + :width: 800px + :alt:TC002 dashboard + +For each test case dashboard. On the top left, we have a dashboard selection, +you can switch to different test cases using this pull-down menu. + +Underneath, we have a pod and scenario selection. +All the pods and scenarios that have ever published test data to the InfluxDB +will be shown here. + +You can check multiple pods or scenarios. + +For each test case, we have a short description and a link to detailed test +case information in Yardstick user guide. + +Underneath, it is the result presentation section. +You can use the time period selection on the top right corner to zoom in or +zoom out the chart. + + +Administration access +===================== + +For a user with administration rights it is easy to update and save any +dashboard configuration. Saved updates immediately take effect and become live. +This may cause issues like: + +- Changes and updates made to the live configuration in Grafana can compromise + existing Grafana content in an unwanted, unpredicted or incompatible way. + Grafana as such is not version controlled, there exists one single Grafana + configuration per dashboard. +- There is a risk several people can disturb each other when doing updates to + the same Grafana dashboard at the same time. + +Any change made by administrator should be careful. + + +Add a dashboard into yardstick grafana +====================================== + +Due to security concern, users that using the public opnfv account are not able +to edit the yardstick grafana directly.It takes a few more steps for a +non-yardstick user to add a custom dashboard into yardstick grafana. + +There are 6 steps to go. + + +.. image:: images/add.png + :width: 800px + :alt: Add a dashboard into yardstick grafana + + +1. You need to build a local influxdb and grafana, so you can do the work + locally. You can refer to How to deploy InfluxDB and Grafana locally wiki + page about how to do this. + +2. Once step one is done, you can fetch the existing grafana dashboard + configuration file from the yardstick repository and import it to your local + grafana. After import is done, you grafana dashboard will be ready to use + just like the community’s dashboard. + +3. The third step is running some test cases to generate test results and + publishing it to your local influxdb. + +4. Now you have some data to visualize in your dashboard. In the fourth step, + it is time to create your own dashboard. You can either modify an existing + dashboard or try to create a new one from scratch. If you choose to modify + an existing dashboard then in the curtain menu of the existing dashboard do + a "Save As..." into a new dashboard copy instance, and then continue doing + all updates and saves within the dashboard copy. + +5. When finished with all Grafana configuration changes in this temporary + dashboard then chose "export" of the updated dashboard copy into a JSON file + and put it up for review in Gerrit, in file /yardstick/dashboard/Yardstick-TCxxx-yyyyyyyyyyyyy. + For instance a typical default name of the file would be "Yardstick-TC001 Copy-1234567891234". + +6. Once you finish your dashboard, the next step is exporting the configuration + file and propose a patch into Yardstick. Yardstick team will review and + merge it into Yardstick repository. After approved review Yardstick team + will do an "import" of the JSON file and also a "save dashboard" as soon as + possible to replace the old live dashboard configuration. + diff --git a/docs/userguide/10-list-of-tcs.rst b/docs/userguide/11-list-of-tcs.rst index 7e8c85433..8798a8f51 100644 --- a/docs/userguide/10-list-of-tcs.rst +++ b/docs/userguide/11-list-of-tcs.rst @@ -49,6 +49,7 @@ Generic NFVI Test Case Descriptions opnfv_yardstick_tc070.rst opnfv_yardstick_tc071.rst opnfv_yardstick_tc072.rst + opnfv_yardstick_tc073.rst opnfv_yardstick_tc075.rst OPNFV Feature Test Cases @@ -62,6 +63,16 @@ H A opnfv_yardstick_tc019.rst opnfv_yardstick_tc025.rst + opnfv_yardstick_tc045.rst + opnfv_yardstick_tc046.rst + opnfv_yardstick_tc047.rst + opnfv_yardstick_tc048.rst + opnfv_yardstick_tc049.rst + opnfv_yardstick_tc050.rst + opnfv_yardstick_tc051.rst + opnfv_yardstick_tc052.rst + opnfv_yardstick_tc053.rst + opnfv_yardstick_tc054.rst IPv6 ---- @@ -87,6 +98,14 @@ Parser opnfv_yardstick_tc040.rst + StorPerf +----------- + +.. toctree:: + :maxdepth: 1 + + opnfv_yardstick_tc074.rst + virtual Traffic Classifier -------------------------- @@ -106,3 +125,4 @@ Templates testcase_description_v2_template Yardstick_task_templates + diff --git a/docs/userguide/images/TC002.png b/docs/userguide/images/TC002.png Binary files differnew file mode 100644 index 000000000..89154efcc --- /dev/null +++ b/docs/userguide/images/TC002.png diff --git a/docs/userguide/images/add.png b/docs/userguide/images/add.png Binary files differnew file mode 100644 index 000000000..a88a1b146 --- /dev/null +++ b/docs/userguide/images/add.png diff --git a/docs/userguide/images/login.png b/docs/userguide/images/login.png Binary files differnew file mode 100644 index 000000000..045e010e4 --- /dev/null +++ b/docs/userguide/images/login.png diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index 0aa112a45..60e1340ac 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -19,6 +19,7 @@ Yardstick Overview 07-installation 08-yardstick_plugin 09-result-store-InfluxDB - 10-list-of-tcs + 10-grafana + 11-list-of-tcs glossary references diff --git a/docs/userguide/opnfv_yardstick_tc043.rst b/docs/userguide/opnfv_yardstick_tc043.rst index b6e557d86..59d7c6993 100644 --- a/docs/userguide/opnfv_yardstick_tc043.rst +++ b/docs/userguide/opnfv_yardstick_tc043.rst @@ -13,7 +13,7 @@ Yardstick Test Case Description TC043 |Network Latency Between NFVI Nodes | | | +--------------+--------------------------------------------------------------+ -|test case id | OPNFV_YARDSTICK_TC043_Latency_between_NFVI_nodes_ | +|test case id | OPNFV_YARDSTICK_TC043_Latency_between_NFVI_nodes | | | measurements | | | | +--------------+--------------------------------------------------------------+ diff --git a/docs/userguide/opnfv_yardstick_tc053.rst b/docs/userguide/opnfv_yardstick_tc053.rst index 8808d12d9..3c6bbc628 100644 --- a/docs/userguide/opnfv_yardstick_tc053.rst +++ b/docs/userguide/opnfv_yardstick_tc053.rst @@ -13,7 +13,7 @@ Yardstick Test Case Description TC053 | | +--------------+--------------------------------------------------------------+ |test case id | OPNFV_YARDSTICK_TC053: OpenStack Controller Load Balance | -| | Service High Availability- | +| | Service High Availability | +--------------+--------------------------------------------------------------+ |test purpose | This test case will verify the high availability of the | | | load balance service(current is HAProxy) that supports | diff --git a/docs/userguide/opnfv_yardstick_tc073.rst b/docs/userguide/opnfv_yardstick_tc073.rst index a6499eabb..ad4526405 100644 --- a/docs/userguide/opnfv_yardstick_tc073.rst +++ b/docs/userguide/opnfv_yardstick_tc073.rst @@ -37,7 +37,7 @@ Yardstick Test Case Description TC073 | | For SLA max_mean_latency is set to 100. | | | | +--------------+--------------------------------------------------------------+ -|test tool | netperf | +|test tool | netperf_ | | | Netperf is a software application that provides network | | | bandwidth testing between two hosts on a network. It | | | supports Unix domain sockets, TCP, SCTP, DLPI and UDP via | diff --git a/docs/userguide/opnfv_yardstick_tc074.rst b/docs/userguide/opnfv_yardstick_tc074.rst index c938f5dfd..92cd51439 100644 --- a/docs/userguide/opnfv_yardstick_tc074.rst +++ b/docs/userguide/opnfv_yardstick_tc074.rst @@ -7,7 +7,7 @@ Yardstick Test Case Description TC074 ************************************* -.. Storperf: https://wiki.opnfv.org/display/storperf/Storperf +.. _Storperf: https://wiki.opnfv.org/display/storperf/Storperf +-----------------------------------------------------------------------------+ |Storperf | @@ -44,7 +44,7 @@ Yardstick Test Case Description TC074 | | * timeout: 600 - maximum allowed job time | | | | +--------------+--------------------------------------------------------------+ -|test tool | Storperf | +|test tool | Storperf_ | | | | | | StorPerf is a tool to measure block and object storage | | | performance in an NFVI. | diff --git a/docs/userguide/references.rst b/docs/userguide/references.rst index 551926135..05729ba75 100644 --- a/docs/userguide/references.rst +++ b/docs/userguide/references.rst @@ -15,25 +15,34 @@ OPNFV * Pharos wiki: https://wiki.opnfv.org/pharos * VTC: https://wiki.opnfv.org/vtc * Yardstick CI: https://build.opnfv.org/ci/view/yardstick/ -* Yardstick and ETSI TST001 presentation: https://wiki.opnfv.org/_media/opnfv_summit_-_bridging_opnfv_and_etsi.pdf -* Yardstick Project presentation: https://wiki.opnfv.org/_media/opnfv_summit_-_yardstick_project.pdf +* Yardstick and ETSI TST001 presentation: https://wiki.opnfv.org/display/yardstick/Yardstick?preview=%2F2925202%2F2925205%2Fopnfv_summit_-_bridging_opnfv_and_etsi.pdf +* Yardstick Project presentation: https://wiki.opnfv.org/display/yardstick/Yardstick?preview=%2F2925202%2F2925208%2Fopnfv_summit_-_yardstick_project.pdf * Yardstick wiki: https://wiki.opnfv.org/yardstick References used in Test Cases ============================= +* cachestat: https://github.com/brendangregg/perf-tools/tree/master/fs * cirros-image: https://download.cirros-cloud.net * cyclictest: https://rt.wiki.kernel.org/index.php/Cyclictest * DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/ * DPDK supported NICs: http://dpdk.org/doc/nics +* fdisk: http://www.tldp.org/HOWTO/Partition/fdisk_partitioning.html * fio: http://www.bluestop.org/fio/HOWTO.txt +* free: http://manpages.ubuntu.com/manpages/trusty/en/man1/free.1.html * iperf3: https://iperf.fr/ +* iostat: http://linux.die.net/man/1/iostat * Lmbench man-pages: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html * Memory bandwidth man-pages: http://manpages.ubuntu.com/manpages/trusty/bw_mem.8.html -* unixbench: https://github.com/kdlucas/byte-unixbench/blob/master/UnixBench * mpstat man-pages: http://manpages.ubuntu.com/manpages/trusty/man1/mpstat.1.html +* netperf: http://www.netperf.org/netperf/training/Netperf.html * pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt +* RAMspeed: http://alasir.com/software/ramspeed/ +* sar: http://linux.die.net/man/1/sar * SR-IOV: https://wiki.openstack.org/wiki/SR-IOV-Passthrough-For-Networking +* Storperf: https://wiki.opnfv.org/display/storperf/Storperf +* unixbench: https://github.com/kdlucas/byte-unixbench/blob/master/UnixBench + Research ======== @@ -46,5 +55,6 @@ Standards ========= * ETSI NFV: http://www.etsi.org/technologies-clusters/technologies/nfv -* ETSI GS-NFV TST 001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/ +* ETSI GS-NFV TST 001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf * RFC2544: https://www.ietf.org/rfc/rfc2544.txt + diff --git a/etc/__init__.py b/etc/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/etc/__init__.py diff --git a/etc/yardstick/nodes/fuel_baremetal/pod.yaml b/etc/yardstick/nodes/fuel_baremetal/pod.yaml new file mode 100644 index 000000000..e83257462 --- /dev/null +++ b/etc/yardstick/nodes/fuel_baremetal/pod.yaml @@ -0,0 +1,43 @@ +--- +# sample config file about the POD information, including the +# name/IP/user/ssh key of Bare Metal and Controllers/Computes +# +# The options of this config file include: +# name: the name of this node +# role: node's role, support role: Master/Controller/Comupte/BareMetal +# ip: the node's IP address +# user: the username for login +# key_filename:the path of the private key file for login +# ipX: the ips of the nodes will be fetched by prepare_env.sh and replaced here + +nodes: +- + name: node1 + role: Controller + ip: ip1 + user: root + key_filename: /root/.ssh/id_rsa +- + name: node2 + role: Controller + ip: ip2 + user: root + key_filename: /root/.ssh/id_rsa +- + name: node3 + role: Controller + ip: ip3 + user: root + key_filename: /root/.ssh/id_rsa +- + name: node4 + role: Compute + ip: ip4 + user: root + key_filename: /root/.ssh/id_rsa +- + name: node5 + role: Compute + ip: ip5 + user: root + key_filename: /root/.ssh/id_rsa diff --git a/etc/yardstick/nodes/fuel_virtual/id_rsa b/etc/yardstick/nodes/fuel_virtual/id_rsa deleted file mode 100644 index 35ac1b5fe..000000000 --- a/etc/yardstick/nodes/fuel_virtual/id_rsa +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA24CCFpmsaPUt9KJsaER/R4IyYvOd0iOXhFuOUCl4nJvBnlXu -D8Pzombgz6bZcHx96ukgmOKq/Bf0tPA4fN733fw/Jjb4t6O4HFVpcBZykcgdnB56 -pqwN108pQZCq8R3EiKU3BgL2nWi9YP94JxsbD8I6vcQVbG7SMeJ0YpQzNYyJ1ig9 -fjff7ROuOc+XVZhG7UtbCz7adGS2/FfGgXz49mLS98pNLMOAUSUtoBog0CveotXM -rWT9OOOpTTihWFspVU1cnT1LGJ+MYVRX2uF7sZASsglwA0kzjlf1nzQ8fGXC3o7D -kFPdM1jtHWf4ah4DSb2e/LnCrwqBgmMyKpV+AQIBIwKCAQEA1TsCB1N0SLOo/EYC -6PIVPia0mqODXmu3wmeRj7NB9zg4be0TJUICncMGRg/L6Z2Bol7PNW56NrgvizKA -BEZP3vUKJR91RK2riT0HVvE8GJaDKfG4+a5z2HjI/dz91EjNjA41c43ZoDncihy9 -3NiAsDkF3Opd9E5lyg8vOzDhSfRwCg2u2HMiGy9/yB+V8x8QJ2vjDzXI1Nn40jAa -azw94wIdkHCrXRbt3z5zNMNRTQtGUD1uUSsTGO2AH4LY/Dtq0kAZw2f4dgfZS8f3 -FGIa2o3wpJfgZWmHS+jWBg1ADZTr45Ur2BpKIo/GcjeTdf8DkNQ2/hy0x3JGBVBE -e5HvYwKBgQDzb5ApXZu7gOm5XOCWC0cNiqQT7VUHQMdc9i7waV862kH7+Jn1nPBj -QsJQhGVrB/vP3PhkGbRHp86QBJiwTpY4m9YQFfvFH47a2NWrlafNyvQ5y37OIsD0 -ib8vYRXQrBFnrkBn+wCbDNTy04v3hbPayN5OC7EwT1g7PbexIpYH7wKBgQDm1Lcr -bvhyQR61FnQjKmX0DOVk6Jve0Pk0PNkzXm7JsnF8U+mJClmgqJOEoaBaj3dGZDTC -TzGLUcZCkZk+2PEv8BcyAd4GNETZYCHEgxDsnDbotZPvlkFCzQr4+6delSv+8zTM -kTgyEf8IW8/4Jy7wIH1UkRgyoXFYin6F1BZpDwKBgQDekeLj/dA2ZzwXMFhOqzml -+xmr0azTbm0hyyOZ+fCq1i2zLG+BeYtTcDyhYxrlg6RmRl9xdpYy4pD4s77NFKaa -KBQr9tePp9MRO0cDRv/SGKTHIHPvqr8LdqArUXMH7cbFMZn4qvk9TY9+7UzFDIcu -boIbeGd8oFCrMRz5uTi2ywKBgQCXsFscisCFmIHktvvcmDRejCG3VwdX6Gk/lbNN -pHSwbfLOC0Gx05n73H4ylhjrDdIJr5Bibo5FnCMzDzjRhz9opRaOk4NF59V452al -tTcB4v+C+vrQpJFJJ6gf9dRiucx0Vq2rAFgg50ExYOfAVENqmQHnHYTuElHMeESD -1IPBYQKBgQDUEnv0htzJjJef2080CMN4RG+Sq2PHBkFlwe6fL52+AVv/zyCqKqy5 -TXRV1tHnMTII68+/NobwG+UC8dCBQSFmbLEqHcLF1+PxtOiSkRotU8rBZAafuMCS -ajH+CdGMwUkMvsPL2PnnX/A6w0PJZM/arpou9qOI1bzxQuL7h43zzw== ------END RSA PRIVATE KEY----- diff --git a/fuel-plugin/deployment_scripts/install.sh b/fuel-plugin/deployment_scripts/install.sh index 76ef433c2..6882f0be2 100755 --- a/fuel-plugin/deployment_scripts/install.sh +++ b/fuel-plugin/deployment_scripts/install.sh @@ -25,6 +25,6 @@ easy_install -U setuptools cd $BIN_HOME -curl http://$HOST:8080/plugins/fuel-plugin-yardstick-0.9/repositories/ubuntu/yardstick.tar.gz | tar xzvf - +curl http://$HOST:8080/plugins/fuel-plugin-yardstick-1.0/repositories/ubuntu/yardstick.tar.gz | tar xzvf - python setup.py develop diff --git a/fuel-plugin/deployment_scripts/puppet/manifests/yardstick-install.pp b/fuel-plugin/deployment_scripts/puppet/manifests/yardstick-install.pp index 7993524d0..82dfff387 100644 --- a/fuel-plugin/deployment_scripts/puppet/manifests/yardstick-install.pp +++ b/fuel-plugin/deployment_scripts/puppet/manifests/yardstick-install.pp @@ -16,7 +16,7 @@ $identity_uri = "${internal_auth_protocol}://${internal_auth_address}: $auth_url = "${identity_uri}/${auth_api_version}" exec { "install yardstick": - command => "curl http://${master_ip}:8080/plugins/fuel-plugin-yardstick-0.9/deployment_scripts/install.sh | bash -s ${master_ip}", + command => "curl http://${master_ip}:8080/plugins/fuel-plugin-yardstick-1.0/deployment_scripts/install.sh | bash -s ${master_ip}", path => "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin"; } diff --git a/fuel-plugin/metadata.yaml b/fuel-plugin/metadata.yaml index 2f6c95281..e9aebaf2a 100644 --- a/fuel-plugin/metadata.yaml +++ b/fuel-plugin/metadata.yaml @@ -3,11 +3,11 @@ name: fuel-plugin-yardstick # Human-readable name for your plugin title: Install Yardstick # Plugin version -version: '0.9.0' +version: '1.0.0' # Description description: Installs Yardstick # Required fuel version -fuel_version: ['9.0'] +fuel_version: ['10.0'] # Specify license of your plugin licenses: ['Apache License Version 2.0'] # Specify author or company name @@ -27,7 +27,7 @@ package_version: '4.0.0' # The plugin is compatible with releases in the list releases: - os: ubuntu - version: mitaka-9.0 + version: newton-10.0 mode: ['ha'] deployment_scripts_path: deployment_scripts/ repository_path: repositories/ubuntu diff --git a/install.sh b/install.sh new file mode 100755 index 000000000..e9b6035d9 --- /dev/null +++ b/install.sh @@ -0,0 +1,44 @@ +# install tools +apt-get update && apt-get install -y \ + wget \ + expect \ + curl \ + git \ + sshpass \ + qemu-utils \ + kpartx \ + libffi-dev \ + libssl-dev \ + python \ + python-dev \ + libxml2-dev \ + libxslt1-dev \ + nginx \ + uwsgi \ + uwsgi-plugin-python \ + supervisor \ + python-setuptools && \ + easy_install -U setuptools + +apt-get -y autoremove && apt-get clean + + +# fit for arm64 +source_file=/etc/apt/sources.list +sed -i -e 's/^deb \([^/[]\)/deb [arch=amd64] \1/g' "${source_file}" +sed -i -e 's/^deb-src /# deb-src /g' "${source_file}" + +sub_source_file=/etc/apt/sources.list.d/yardstick.list +touch "${sub_source_file}" +echo -e "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty main universe multiverse restricted +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main universe multiverse restricted +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" > "${sub_source_file}" +echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf +dpkg --add-architecture arm64 +apt-get install -y qemu-user-static libc6:arm64 + +# install yardstick + dependencies +easy_install -U pip +pip install -r requirements.txt +pip install . diff --git a/tests/ci/requirements.txt b/requirements.txt index 4d1a16993..e391c9210 100644 --- a/tests/ci/requirements.txt +++ b/requirements.txt @@ -77,3 +77,10 @@ unicodecsv==0.14.1 unittest2==1.1.0 warlock==1.2.0 wrapt==1.10.6 +flask==0.11.1 +flask-restful==0.3.5 +influxdb==3.0.0 +pyroute2==0.4.10 +docker-py==1.10.6 +flasgger==0.5.13 +flask-restful-swagger==0.19 diff --git a/samples/background-task.yaml b/samples/background-task.yaml index a844c2d7f..11cfdd567 100644 --- a/samples/background-task.yaml +++ b/samples/background-task.yaml @@ -39,7 +39,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/cachestat.yaml b/samples/cachestat.yaml index 5786efa38..d736793e3 100644 --- a/samples/cachestat.yaml +++ b/samples/cachestat.yaml @@ -18,7 +18,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/computecapacity.yaml b/samples/computecapacity.yaml index 006b3ef3d..ae527d2ca 100644 --- a/samples/computecapacity.yaml +++ b/samples/computecapacity.yaml @@ -1,8 +1,12 @@ --- # Sample benchmark task config file -# Measure compute capacity and scale. -# Including number of cores, number of threads, available memory size and -# cache size. +# compute capacity and scale. + +# the results have +# number of CPUs, number of physical cores in a single CPU +# number of logical cores, total memory size +# cache size per CPU, total cache size +# HT (Hyper-Thread) support status, 1 for open, 0 for close schema: "yardstick:task:0.1" diff --git a/samples/cyclictest.yaml b/samples/cyclictest.yaml index cb85decb1..eaf74893e 100644 --- a/samples/cyclictest.yaml +++ b/samples/cyclictest.yaml @@ -33,7 +33,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu servers: diff --git a/samples/fio-template.yaml b/samples/fio-template.yaml index ba710d95f..00c35ce23 100644 --- a/samples/fio-template.yaml +++ b/samples/fio-template.yaml @@ -28,7 +28,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu servers: diff --git a/samples/fio.yaml b/samples/fio.yaml index e1f5e6d2d..5ccbc1954 100644 --- a/samples/fio.yaml +++ b/samples/fio.yaml @@ -37,7 +37,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu servers: diff --git a/samples/iperf3-jitter.yaml b/samples/iperf3-jitter.yaml index c211571d0..366a57152 100644 --- a/samples/iperf3-jitter.yaml +++ b/samples/iperf3-jitter.yaml @@ -23,7 +23,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/iperf3.yaml b/samples/iperf3.yaml index 72f260942..6741c767e 100644 --- a/samples/iperf3.yaml +++ b/samples/iperf3.yaml @@ -21,7 +21,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/lmbench.yaml b/samples/lmbench.yaml index 311770c42..595a393b7 100644 --- a/samples/lmbench.yaml +++ b/samples/lmbench.yaml @@ -48,7 +48,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/lmbench_cache.yaml b/samples/lmbench_cache.yaml index 7a22cf15f..bf5086b3c 100644 --- a/samples/lmbench_cache.yaml +++ b/samples/lmbench_cache.yaml @@ -26,7 +26,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/memload.yaml b/samples/memload.yaml index 87d727707..5e988986a 100644 --- a/samples/memload.yaml +++ b/samples/memload.yaml @@ -20,7 +20,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/netperf.yaml b/samples/netperf.yaml index 4d7f7a798..0dd56348b 100755 --- a/samples/netperf.yaml +++ b/samples/netperf.yaml @@ -46,7 +46,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/netperf_bottlenecks.yaml b/samples/netperf_bottlenecks.yaml new file mode 100644 index 000000000..4b6348109 --- /dev/null +++ b/samples/netperf_bottlenecks.yaml @@ -0,0 +1,43 @@ +--- +# measure network latency and throughput using netperf +# This test case is suite for bottlenecks project. +# This test case is from TC073 +# we have did some parameters support + +schema: "yardstick:task:0.1" + +{% set host = host or "node1.LF" %} +{% set target = target or "node2.LF" %} +{% set pod_info = pod_info or "etc/yardstick/nodes/compass_sclab_virtual/pod.yaml" %} +{% set tx_msg_size = tx_msg_size or "65536" %} +{% set rx_msg_size = rx_msg_size or "87380" %} +{% set test_time = test_time or "20" %} +{% set out_opt = out_opt or "THROUGHPUT,THROUGHPUT_UNITS,MEAN_LATENCY,LOCAL_CPU_UTIL,REMOTE_CPU_UTIL,LOCAL_TRANSPORT_RETRANS" %} + +scenarios: +- + type: NetperfNode + options: + testname: 'TCP_STREAM' + send_msg_size: {{tx_msg_size}} + recv_msg_size: {{rx_msg_size}} + duration: {{test_time}} + output_opt: {{out_opt}} + + host: {{host}} + target: {{target}} + + runner: + type: Iteration + iterations: 1 + interval: 1 + run_step: 'setup,run' + + sla: + mean_latency: 100 + action: monitor + +context: + type: Node + name: LF + file: {{pod_info}} diff --git a/samples/netutilization.yaml b/samples/netutilization.yaml index 598a5af15..794342d29 100644 --- a/samples/netutilization.yaml +++ b/samples/netutilization.yaml @@ -19,7 +19,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/perf.yaml b/samples/perf.yaml index 541f846e7..b8979b511 100644 --- a/samples/perf.yaml +++ b/samples/perf.yaml @@ -30,7 +30,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/ping6.yaml b/samples/ping6.yaml index 6f2c93d05..2fb99dfb4 100644 --- a/samples/ping6.yaml +++ b/samples/ping6.yaml @@ -8,6 +8,7 @@ scenarios: type: Ping6 options: packetsize: 200 + ping_count: 5 host: host1,host2,host3,host4,host5 nodes: host1: node1.IPV6 diff --git a/samples/pktgen.yaml b/samples/pktgen.yaml index ddafb27bd..6acb8ab92 100644 --- a/samples/pktgen.yaml +++ b/samples/pktgen.yaml @@ -38,7 +38,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/ramspeed.yaml b/samples/ramspeed.yaml index 7e1b1aa8c..e754fc9fa 100644 --- a/samples/ramspeed.yaml +++ b/samples/ramspeed.yaml @@ -25,7 +25,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/samples/unixbench.yaml b/samples/unixbench.yaml index 825fd767f..b7ab88190 100644 --- a/samples/unixbench.yaml +++ b/samples/unixbench.yaml @@ -21,7 +21,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu @@ -23,31 +23,16 @@ setup( 'resources/files/*', 'resources/scripts/install/*.bash', 'resources/scripts/remove/*.bash' + ], + 'etc': [ + 'yardstick/nodes/*/*.yaml' + ], + 'tests': [ + 'opnfv/*/*.yaml', + 'ci/*.sh' ] }, url="https://www.opnfv.org", - install_requires=["backport_ipaddress", # remove with python3 - "coverage>=3.6", - "flake8", - "Jinja2>=2.6", - "lxml", - "PyYAML>=3.10", - "pbr<2.0,>=1.3", - "python-openstackclient>=2.1.0", - "python-glanceclient>=0.12.0", - "python-heatclient>=0.2.12", - "python-keystoneclient>=0.11.1", - "python-neutronclient>=2.3.9", - "python-novaclient>=2.24.1", - "mock>=1.0.1", # remove with python3 - "paramiko", - "netifaces", - "scp", - "six", - "testrepository>=0.0.18", - "testtools>=1.4.0", - "nose" - ], extras_require={ 'plot': ["matplotlib>=1.4.2"] }, @@ -57,5 +42,9 @@ setup( 'yardstick-plot=yardstick.plot.plotter:main [plot]' ], }, - scripts=['tools/yardstick-img-modify'] + scripts=[ + 'tools/yardstick-img-modify', + 'tools/yardstick-img-lxd-modify', + 'tools/yardstick-img-dpdk-modify' + ] ) diff --git a/tests/ci/clean_images.sh b/tests/ci/clean_images.sh new file mode 100755 index 000000000..b1942160b --- /dev/null +++ b/tests/ci/clean_images.sh @@ -0,0 +1,29 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB, Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Clean the images and flavor before running yardstick test suites. + +cleanup() +{ + echo + echo "========== Cleanup ==========" + + if ! glance image-list; then + return + fi + + for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-image -e Ubuntu-14.04 \ + -e yardstick-vivid-kernel | awk '{print $2}'); do + echo "Deleting image $image..." + glance image-delete $image || true + done + + nova flavor-delete yardstick-flavor &> /dev/null || true +} diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh new file mode 100755 index 000000000..405d72076 --- /dev/null +++ b/tests/ci/load_images.sh @@ -0,0 +1,237 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB, Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Set up the environment to run yardstick test suites. + +set -e + +YARD_IMG_ARCH=amd64 +export YARD_IMG_ARCH + +if ! grep -q "Defaults env_keep += \"YARD_IMG_ARCH\"" "/etc/sudoers"; then + sudo echo "Defaults env_keep += \"YARD_IMG_ARCH YARDSTICK_REPO_DIR\"" >> /etc/sudoers +fi + +ARCH_SCRIPT="test -f /etc/fuel_openstack_arch && grep -q arm64 /etc/fuel_openstack_arch" +if [ "$INSTALLER_TYPE" == "fuel" ]; then + sshpass -p r00tme ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -l root $INSTALLER_IP "${ARCH_SCRIPT}" && YARD_IMG_ARCH=arm64 +fi + +UCA_HOST="cloud-images.ubuntu.com" +if [ $YARD_IMG_ARCH = "arm64" ]; then + export VIVID_IMG_URL="http://${UCA_HOST}/vivid/current/vivid-server-cloudimg-arm64.tar.gz" + if ! grep -q "Defaults env_keep += \"VIVID_IMG_URL\"" "/etc/sudoers"; then + sudo echo "Defaults env_keep += \"VIVID_IMG_URL\"" >> /etc/sudoers + fi +fi + +build_yardstick_image() +{ + echo + echo "========== Build yardstick cloud image ==========" + + if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then + local cmd="sudo $(which yardstick-img-lxd-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh" + + # Build the image. Retry once if the build fails + $cmd || $cmd + + if [ ! -f $RAW_IMAGE ]; then + echo "Failed building RAW image" + exit 1 + fi + else + local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh" + + # Build the image. Retry once if the build fails + $cmd || $cmd + + if [ ! -f $QCOW_IMAGE ]; then + echo "Failed building QCOW image" + exit 1 + fi + fi +} + +load_yardstick_image() +{ + echo + echo "========== Loading yardstick cloud image ==========" + EXTRA_PARAMS="" + if [ $YARD_IMG_ARCH = "arm64" ]; then + VIVID_IMAGE="/tmp/vivid-server-cloudimg-arm64.tar.gz" + VIVID_KERNEL="/tmp/vivid-server-cloudimg-arm64-vmlinuz-generic" + cd /tmp + if [ ! -f $VIVID_IMAGE ]; then + wget $VIVID_IMG_URL + fi + if [ ! -f $VIVID_KERNEL ]; then + tar zxf $VIVID_IMAGE $(basename $VIVID_KERNEL) + fi + create_vivid_kernel=$(glance --os-image-api-version 1 image-create \ + --name yardstick-vivid-kernel \ + --is-public true --disk-format qcow2 \ + --container-format bare \ + --file $VIVID_KERNEL) + + GLANCE_KERNEL_ID=$(echo "$create_vivid_kernel" | grep " id " | awk '{print $(NF-1)}') + if [ -z "$GLANCE_KERNEL_ID" ]; then + echo 'Failed uploading kernel to cloud'. + exit 1 + fi + + command_line="root=/dev/vdb1 console=tty0 console=ttyS0 console=ttyAMA0 rw" + + EXTRA_PARAMS="--property kernel_id=$GLANCE_KERNEL_ID --property os_command_line=\"$command_line\"" + + rm -f $VIVID_KERNEL $VIVID_IMAGE + cd $YARDSTICK_REPO_DIR + fi + + # VPP requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then + EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large" + fi + + if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then + output=$(eval glance --os-image-api-version 1 image-create \ + --name yardstick-image \ + --is-public true --disk-format root-tar \ + --container-format bare \ + $EXTRA_PARAMS \ + --file $RAW_IMAGE) + else + output=$(eval glance --os-image-api-version 1 image-create \ + --name yardstick-image \ + --is-public true --disk-format qcow2 \ + --container-format bare \ + $EXTRA_PARAMS \ + --file $QCOW_IMAGE) + fi + + echo "$output" + + GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') + + if [ -z "$GLANCE_IMAGE_ID" ]; then + echo 'Failed uploading image to cloud'. + exit 1 + fi + + if [ "$DEPLOY_SCENARIO" == *"-lxd-"* ]; then + sudo rm -f $RAW_IMAGE + else + sudo rm -f $QCOW_IMAGE + fi + + echo "Glance image id: $GLANCE_IMAGE_ID" +} + +load_cirros_image() +{ + echo + echo "========== Loading cirros cloud image ==========" + + local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img + + EXTRA_PARAMS="" + # VPP requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then + EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large" + fi + + output=$(glance image-create \ + --name cirros-0.3.3 \ + --disk-format qcow2 \ + --container-format bare \ + $EXTRA_PARAMS \ + --file $image_file) + echo "$output" + + CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') + if [ -z "$CIRROS_IMAGE_ID" ]; then + echo 'Failed uploading cirros image to cloud'. + exit 1 + fi + + echo "Cirros image id: $CIRROS_IMAGE_ID" +} + +load_ubuntu_image() +{ + echo + echo "========== Loading ubuntu cloud image ==========" + + local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img + + EXTRA_PARAMS="" + # VPP requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then + EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large" + fi + + output=$(glance image-create \ + --name Ubuntu-14.04 \ + --disk-format qcow2 \ + --container-format bare \ + $EXTRA_PARAMS \ + --file $ubuntu_image_file) + echo "$output" + + UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') + + if [ -z "$UBUNTU_IMAGE_ID" ]; then + echo 'Failed uploading UBUNTU image to cloud'. + exit 1 + fi + + echo "Ubuntu image id: $UBUNTU_IMAGE_ID" +} + +create_nova_flavor() +{ + if ! nova flavor-list | grep -q yardstick-flavor; then + echo + echo "========== Create nova flavor ==========" + # Create the nova flavor used by some sample test cases + nova flavor-create yardstick-flavor 100 512 3 1 + # DPDK-enabled OVS requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then + nova flavor-key yardstick-flavor set hw:mem_page_size=large + fi + # VPP requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then + nova flavor-key yardstick-flavor set hw:mem_page_size=large + fi + fi +} + +main() +{ + QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-image.img" + RAW_IMAGE="/tmp/workspace/yardstick/yardstick-image.tar.gz" + + build_yardstick_image + load_yardstick_image + if [ $YARD_IMG_ARCH = "arm64" ]; then + sed -i 's/image: cirros-0.3.3/image: TestVM/g' tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml \ + samples/ping.yaml + #We have overlapping IP with the real network + for filename in tests/opnfv/test_cases/*; do + sed -i "s/cidr: '10.0.1.0\/24'/cidr: '10.3.1.0\/24'/g" $filename + done + else + load_cirros_image + load_ubuntu_image + fi + create_nova_flavor +} + +main diff --git a/tests/ci/prepare_env.sh b/tests/ci/prepare_env.sh index d9f8257ae..be59b7f37 100755 --- a/tests/ci/prepare_env.sh +++ b/tests/ci/prepare_env.sh @@ -15,7 +15,6 @@ : ${NODE_NAME:='unknown'} : ${EXTERNAL_NETWORK:='admin_floating_net'} - # Extract network name from EXTERNAL_NETWORK # e.g. EXTERNAL_NETWORK='ext-net;flat;192.168.0.2;192.168.0.253;192.168.0.1;192.168.0.0/24' export EXTERNAL_NETWORK=$(echo $EXTERNAL_NETWORK | cut -f1 -d \;) @@ -83,5 +82,33 @@ if [ "$INSTALLER_TYPE" == "fuel" ]; then echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." sshpass -p r00tme scp 2>/dev/null $ssh_options \ root@${INSTALLER_IP}:~/.ssh/id_rsa /root/.ssh/id_rsa &> /dev/null -fi + sshpass -p r00tme ssh 2>/dev/null $ssh_options \ + root@${INSTALLER_IP} fuel node>fuel_node + + # update fuel node id and ip info according to the CI env + controller_IDs=($(cat fuel_node|grep controller|awk '{print $1}')) + compute_IDs=($(cat fuel_node|grep compute|awk '{print $1}')) + controller_ips=($(cat fuel_node|grep controller|awk '{print $10}')) + compute_ips=($(cat fuel_node|grep compute|awk '{print $10}')) + + pod_yaml="./etc/yardstick/nodes/fuel_baremetal/pod.yaml" + node_line_num=($(grep -n node[1-5] $pod_yaml | awk -F: '{print $1}')) + + if [[ ${controller_ips[0]} ]]; then + sed -i "${node_line_num[0]}s/node1/node${controller_IDs[0]}/;s/ip1/${controller_ips[0]}/" $pod_yaml; + fi + if [[ ${controller_ips[1]} ]]; then + sed -i "${node_line_num[1]}s/node2/node${controller_IDs[1]}/;s/ip2/${controller_ips[1]}/" $pod_yaml; + fi + if [[ ${controller_ips[2]} ]]; then + sed -i "${node_line_num[2]}s/node3/node${controller_IDs[2]}/;s/ip3/${controller_ips[2]}/" $pod_yaml; + fi + if [[ ${compute_ips[0]} ]]; then + sed -i "${node_line_num[3]}s/node4/node${compute_IDs[0]}/;s/ip4/${compute_ips[0]}/" $pod_yaml; + fi + if [[ ${compute_ips[1]} ]]; then + sed -i "${node_line_num[4]}s/node5/node${compute_IDs[1]}/;s/ip5/${compute_ips[1]}/" $pod_yaml; + fi + +fi diff --git a/tests/ci/report_config.yaml b/tests/ci/report_config.yaml new file mode 100644 index 000000000..5346e608b --- /dev/null +++ b/tests/ci/report_config.yaml @@ -0,0 +1,7 @@ +reporting: + - + name: apex + scenario: + - + os-nosdn-ovs-noha + diff --git a/tests/ci/yardstick-verify b/tests/ci/yardstick-verify index eafadf987..7644c96c4 100755 --- a/tests/ci/yardstick-verify +++ b/tests/ci/yardstick-verify @@ -71,23 +71,6 @@ done shift $[OPTIND - 1] TEST_SUITES=$@ -cleanup() -{ - echo - echo "========== Cleanup ==========" - - if ! glance image-list; then - return - fi - - for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-trusty-server -e Ubuntu-14.04 | awk '{print $2}'); do - echo "Deleting image $image..." - glance image-delete $image || true - done - - nova flavor-delete yardstick-flavor &> /dev/null || true -} - exitcode="" error_exit() @@ -151,105 +134,23 @@ remove_storperf() fi } -build_yardstick_image() -{ - echo - echo "========== Build yardstick cloud image ==========" - - local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh" - - # Build the image. Retry once if the build fails. - $cmd || $cmd - - if [ ! -f $QCOW_IMAGE ]; then - echo "Failed building QCOW image" - exit 1 - fi -} - -create_nova_flavor() -{ - if ! nova flavor-list | grep -q yardstick-flavor; then - echo - echo "========== Create nova flavor ==========" - # Create the nova flavor used by some sample test cases - nova flavor-create yardstick-flavor 100 512 3 1 - # DPDK-enabled OVS requires guest memory to be backed by large pages - if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then - nova flavor-key yardstick-flavor set hw:mem_page_size=large - fi - fi -} - -load_cirros_image() -{ - echo - echo "========== Loading cirros cloud image ==========" - - local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img - - output=$(glance image-create \ - --name cirros-0.3.3 \ - --disk-format qcow2 \ - --container-format bare \ - --file $image_file) - echo "$output" - - CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') - if [ -z "$CIRROS_IMAGE_ID" ]; then - echo 'Failed uploading cirros image to cloud'. - exit 1 - fi - - echo "Cirros image id: $CIRROS_IMAGE_ID" -} - -load_ubuntu_image() -{ - echo - echo "========== Loading ubuntu cloud image ==========" - - local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img - - output=$(glance image-create \ - --name Ubuntu-14.04 \ - --disk-format qcow2 \ - --container-format bare \ - --file $ubuntu_image_file) - echo "$output" - - UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') - - if [ -z "$UBUNTU_IMAGE_ID" ]; then - echo 'Failed uploading UBUNTU image to cloud'. - exit 1 - fi - - echo "Ubuntu image id: $UBUNTU_IMAGE_ID" -} +report(){ -load_yardstick_image() -{ echo - echo "========== Loading yardstick cloud image ==========" - - output=$(glance --os-image-api-version 1 image-create \ - --name yardstick-trusty-server \ - --is-public true --disk-format qcow2 \ - --container-format bare \ - --file $QCOW_IMAGE) - echo "$output" - - GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') - - if [ -z "$GLANCE_IMAGE_ID" ]; then - echo 'Failed uploading image to cloud'. - exit 1 - fi - - sudo rm -f $QCOW_IMAGE - - echo "Glance image id: $GLANCE_IMAGE_ID" + echo "========== Reporting Status ==========" + curl -i -H 'content-type: application/json' -X POST -d \ + "{\"project_name\": \"yardstick\", + \"case_name\": \"scenario_status\", + \"pod_name\":\"${NODE_NAME}\", + \"installer\":\"${INSTALLER_TYPE}\", + \"version\":\"$(basename ${YARDSTICK_BRANCH})\", + \"scenario\":\"${DEPLOY_SCENARIO}\", + \"description\": \"yardstick ci scenario status\", + \"criteria\":\"$1\", + \"start_date\":\"$2\", + \"stop_date\":\"$3\", + \"details\":\"\"}" \ + ${DISPATCHER_HTTP_TARGET} } run_test() @@ -259,9 +160,9 @@ run_test() mkdir -p /etc/yardstick - cat << EOF >> /etc/yardstick/yardstick.conf + cat << EOF > /etc/yardstick/yardstick.conf [DEFAULT] -debug = True +debug = False dispatcher = ${DISPATCHER_TYPE} [dispatcher_file] @@ -296,7 +197,7 @@ EOF # Mark the test suite failed but continue # running the remaining test suites. - (( failed++ )) + (( ++failed )) fi if [ ${DISPATCHER_TYPE} = file ]; then echo "---------------------------" @@ -313,23 +214,13 @@ EOF - local sceanrio_status="SUCCESS" + local scenario_status="SUCCESS" if [ $failed -gt 0 ]; then scenario_status="FAILED" fi - curl -i -H 'content-type: application/json' -X POST -d \ - "{\"project_name\": \"yardstick\", - \"case_name\": \"scenario_status\", - \"pod_name\":\"${NODE_NAME}\", - \"installer\":\"${INSTALLER_TYPE}\", - \"version\":\"${YARDSTICK_BRANCH}\", - \"scenario\":\"${DEPLOY_SCENARIO}\", - \"description\": \"yardstick ci scenario status\", - \"start_date\":\"${start_date}\", - \"stop_date\":\"${stop_date}\", - \"details\":\"${sceanrio_status}\"}" \ - ${DISPATCHER_HTTP_TARGET} + + report $scenario_status $start_date $stop_date if [ $failed -gt 0 ]; then echo "---------------------------" @@ -427,18 +318,13 @@ main() # install yardstick install_yardstick + source $YARDSTICK_REPO_DIR/tests/ci/clean_images.sh + cleanup trap "error_exit" EXIT SIGTERM - QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img" - - build_yardstick_image - load_yardstick_image - load_cirros_image - load_ubuntu_image - create_nova_flavor - + source $YARDSTICK_REPO_DIR/tests/ci/load_images.sh install_storperf run_test remove_storperf diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc001.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc001.yaml index 899ee963c..aa2980f69 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc001.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc001.yaml @@ -31,7 +31,7 @@ scenarios: context: name: yardstick - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml index 1942bb54f..3c7b988c0 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml @@ -2,6 +2,7 @@ # measure network latency using ping schema: "yardstick:task:0.1" +{% set image = image or "cirros-0.3.3" %} scenarios: {% for i in range(2) %} - @@ -23,8 +24,8 @@ scenarios: context: name: demo - image: cirros-0.3.3 - flavor: m1.tiny + image: {{image}} + flavor: yardstick-flavor user: cirros placement_groups: diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc005.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc005.yaml index 6e50157fc..732d73af7 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc005.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc005.yaml @@ -35,7 +35,7 @@ scenarios: context: name: yardstick-TC005 - image: yardstick-trusty-server + image: yardstick-image flavor: m1.small user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml index 1cec80ff6..a2f5f3adc 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml @@ -37,7 +37,7 @@ scenarios: context: name: yardstick-TC008 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc009.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc009.yaml index 82a55d751..f9fa1b778 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc009.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc009.yaml @@ -32,7 +32,7 @@ scenarios: context: name: yardstick-TC009 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc010.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc010.yaml index aeb18543e..f64968cb1 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc010.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc010.yaml @@ -25,7 +25,7 @@ scenarios: context: name: yardstick-TC010 - image: yardstick-trusty-server + image: yardstick-image flavor: m1.small user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc011.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc011.yaml index 5d21e2814..4cd3119bb 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc011.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc011.yaml @@ -23,7 +23,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc012.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc012.yaml index 3bdb8cb9a..a86943a51 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc012.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc012.yaml @@ -26,7 +26,7 @@ scenarios: context: name: demo - image: yardstick-trusty-server + image: yardstick-image flavor: m1.small user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc014.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc014.yaml index 648657f22..1ac0f2961 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc014.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc014.yaml @@ -19,7 +19,7 @@ scenarios: context: name: yardstick-TC014 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu @@ -29,4 +29,4 @@ context: networks: test: - cidr: '10.0.1.0/24'
\ No newline at end of file + cidr: '10.0.1.0/24' diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc027.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc027.yaml index 544118869..5032f3de3 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc027.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc027.yaml @@ -3,13 +3,18 @@ # Measure IPV6 network latency using ping6 schema: "yardstick:task:0.1" +{% set openrc = openrc or "/opt/admin-openrc.sh" %} +{% set external_network = external_network or "ext-net" %} {% set pod_info = pod_info or "etc/yardstick/nodes/compass_sclab_physical/pod.yaml" %} scenarios: - type: Ping6 options: packetsize: 56 + ping_count: 5 host: host1,host2,host3,host4,host5 + openrc: {{openrc}} + external_network: {{external_network}} nodes: host1: node1.IPV6 host2: node2.IPV6 @@ -25,14 +30,9 @@ scenarios: max_rtt: 30 action: monitor -precondition: - installer_type: compass - deploy_scenarios: os-nosdn - pod_name: huawei-pod1 context: type: Node name: IPV6 file: {{pod_info}} - diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml index 5e2177a6e..cd42098d2 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml @@ -64,7 +64,7 @@ scenarios: context: name: yardstick-TC037 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc038.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc038.yaml index 128f94045..c2e5b4028 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc038.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc038.yaml @@ -64,7 +64,7 @@ scenarios: context: name: yardstick-TC038 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc055.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc055.yaml index b43e56665..54fc965c6 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc055.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc055.yaml @@ -1,6 +1,13 @@ --- # Yardstick TC055 config file -# Collect hardware specification from /proc/cpuinfo +# Collect hardware specification from /proc/cpuinfo /proc/meminfo +# compute capacity and scale. + +# the results have +# number of CPUs, number of physical cores in a single CPU +# number of logical cores, total memory size +# cache size per CPU, total cache size +# HT (Hyper-Thread) support status, 1 for open, 0 for close schema: "yardstick:task:0.1" {% set host = host or "node5.yardstick-TC055" %} diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc063.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc063.yaml index 9da889847..981783765 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc063.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc063.yaml @@ -1,3 +1,4 @@ +--- # Yardstick TC063 config file # Measure disk size, block size and disk utilization using fdisk and iostat diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc069.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc069.yaml index 637e160c6..dcc34d80d 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc069.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc069.yaml @@ -25,7 +25,7 @@ scenarios: context: name: yardstick-TC069 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc070.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc070.yaml index 28b28b9ab..931587b5b 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc070.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc070.yaml @@ -66,7 +66,7 @@ scenarios: context: name: yardstick-TC070 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc071.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc071.yaml index 644010916..6f006eeaf 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc071.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc071.yaml @@ -64,7 +64,7 @@ scenarios: context: name: yardstick-TC071 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc072.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc072.yaml index f3e6d4c40..e1fa33173 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc072.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc072.yaml @@ -66,7 +66,7 @@ scenarios: context: name: yardstick-TC072 - image: yardstick-trusty-server + image: yardstick-image flavor: yardstick-flavor user: ubuntu diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc075.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc075.yaml index d4a978c1d..fb5a12e8a 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc075.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc075.yaml @@ -1,8 +1,7 @@ --- # Yardstick TC075 config file -# Measure compute capacity and scale. -# Including number of cores, number of threads, available memory size and -# cache size. +# Measure network capacity and scale. +# Measure number of connections, number of frames received schema: "yardstick:task:0.1" {% set host = host or "node1.LF" %} diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml new file mode 100644 index 000000000..187e10988 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-fdio-noha_daily.yaml @@ -0,0 +1,30 @@ +--- +# FDS suite + +schema: "yardstick:suite:0.1" + +name: "os-nosdn-fdio-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc001.yaml +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc006.yaml +- + file_name: opnfv_yardstick_tc007.yaml +- + file_name: opnfv_yardstick_tc008.yaml +- + file_name: opnfv_yardstick_tc009.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc020.yaml +- + file_name: opnfv_yardstick_tc021.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc038.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-ha_daily.yaml index eb1226f80..29235b6f6 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-kvm-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-noha_daily.yaml index 02fb31e47..fd48cadb1 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-kvm-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs-ha_daily.yaml index 27accf49c..b488505af 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-kvm_ovs-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-kvm_ovs-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-ha_daily.yaml index cbb2069f9..01591b9d2 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-lxd-ha daily task suite schema: "yardstick:suite:0.1" @@ -8,6 +8,12 @@ test_cases_dir: "tests/opnfv/test_cases/" test_cases: - file_name: opnfv_yardstick_tc002.yaml + constraint: + installer: joid + pod: intel-pod5,intel-pod6 + task_args: + intel-pod5: '{"image": "Cirros LXC 0.3"}' + intel-pod6: '{"image": "Cirros LXC 0.3"}' - file_name: opnfv_yardstick_tc005.yaml - diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-noha_daily.yaml index cbbf8c13e..e1f945b9c 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-lxd-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-lxd-noha daily task suite schema: "yardstick:suite:0.1" @@ -8,6 +8,12 @@ test_cases_dir: "tests/opnfv/test_cases/" test_cases: - file_name: opnfv_yardstick_tc002.yaml + constraint: + installer: joid + pod: intel-pod5,intel-pod6 + task_args: + intel-pod5: '{"image": "Cirros LXC 0.3"}' + intel-pod6: '{"image": "Cirros LXC 0.3"}' - file_name: opnfv_yardstick_tc005.yaml - diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml index ebe7a0513..44c8494d1 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-nofeature-ha daily task suite schema: "yardstick:suite:0.1" @@ -19,13 +19,6 @@ test_cases: - file_name: opnfv_yardstick_tc014.yaml - - file_name: opnfv_yardstick_tc027.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml"}' -- file_name: opnfv_yardstick_tc037.yaml - file_name: opnfv_yardstick_tc043.yaml @@ -111,4 +104,14 @@ test_cases: task_args: huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", "host": "node1.LF"}' +- + file_name: opnfv_yardstick_tc027.yaml + constraint: + installer: compass,fuel + pod: huawei-pod1,lf-pod2,ericsson-pod3,ericsson-pod4 + task_args: + huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml"}' + lf-pod2: '{"pod_info": "etc/yardstick/nodes/fuel_baremetal/pod.yaml", "openrc":"/root/openrc", "external_network":"admin_floating_net"}' + ericsson-pod3: '{"pod_info": "etc/yardstick/nodes/fuel_baremetal/pod.yaml", "openrc":"/root/openrc", "external_network":"admin_floating_net"}' + ericsson-pod4: '{"pod_info": "etc/yardstick/nodes/fuel_baremetal/pod.yaml", "openrc":"/root/openrc", "external_network":"admin_floating_net"}' diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-noha_daily.yaml index 567e8bf73..e85a9788f 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-nofeature-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-ha_daily.yaml index 6cf5b38d3..a61d8242c 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-ovs-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-noha_daily.yaml index 9e5074fc6..6c91a3337 100644 --- a/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-nosdn-ovs-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-nosdn-ovs-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-ha_daily.yaml index 7106a1335..9ea030a40 100644 --- a/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-ocl-nofeature-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-noha_daily.yaml index 42781a841..e2f07650c 100644 --- a/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-ocl-nofeature-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-ocl-nofeature-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-ha_daily.yaml index 639e18e85..cd9c29268 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-bgpvpn-ha daily task suite schema: "yardstick:suite:0.1" @@ -7,40 +7,10 @@ name: "os-odl_l2-bgpvpn-ha" test_cases_dir: "tests/opnfv/test_cases/" test_cases: - - file_name: opnfv_yardstick_tc002.yaml -- - file_name: opnfv_yardstick_tc005.yaml -- - file_name: opnfv_yardstick_tc010.yaml -- - file_name: opnfv_yardstick_tc011.yaml -- - file_name: opnfv_yardstick_tc012.yaml -- - file_name: opnfv_yardstick_tc014.yaml -- - file_name: opnfv_yardstick_tc037.yaml -- - file_name: opnfv_yardstick_tc055.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node5.yardstick-TC055"}' -- - file_name: opnfv_yardstick_tc063.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node5.yardstick-TC063"}' -- - file_name: opnfv_yardstick_tc075.yaml + file_name: opnfv_yardstick_tc043.yaml constraint: - installer: compass - pod: huawei-pod1 + installer: fuel + pod: ericsson-pod2 task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node1.LF"}' + ericsson-pod2: '{"pod_info": "etc/yardstick/nodes/fuel_baremetal/pod.yaml", + "host": "node1.LF","target": "node2.LF"}' diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-noha_daily.yaml index 372041fd6..0d4113e59 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-bgpvpn-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-bgpvpn-noha daily task suite schema: "yardstick:suite:0.1" @@ -7,32 +7,10 @@ name: "os-odl_l2-bgpvpn-noha" test_cases_dir: "tests/opnfv/test_cases/" test_cases: - - file_name: opnfv_yardstick_tc002.yaml -- - file_name: opnfv_yardstick_tc005.yaml -- - file_name: opnfv_yardstick_tc010.yaml -- - file_name: opnfv_yardstick_tc011.yaml -- - file_name: opnfv_yardstick_tc012.yaml -- - file_name: opnfv_yardstick_tc014.yaml -- - file_name: opnfv_yardstick_tc037.yaml -- - file_name: opnfv_yardstick_tc055.yaml - constraint: - installer: compass - pod: huawei-pod1 - task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node5.yardstick-TC055"}' -- - file_name: opnfv_yardstick_tc063.yaml + file_name: opnfv_yardstick_tc043.yaml constraint: - installer: compass - pod: huawei-pod1 + installer: fuel + pod: ericsson-pod2 task_args: - huawei-pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml", - "host": "node5.yardstick-TC063"}' + ericsson-pod2: '{"pod_info": "etc/yardstick/nodes/fuel_baremetal/pod.yaml", + "host": "node1.LF","target": "node2.LF"}' diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-ha_daily.yaml new file mode 100644 index 000000000..2d7397a96 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-ha_daily.yaml @@ -0,0 +1,30 @@ +--- +# os-odl_l2-fdio-ha daily task suite + +schema: "yardstick:suite:0.1" + +name: "os-odl_l2-fdio-ha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc069.yaml +- + file_name: opnfv_yardstick_tc070.yaml +- + file_name: opnfv_yardstick_tc071.yaml +- + file_name: opnfv_yardstick_tc072.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml new file mode 100644 index 000000000..3b7fe80e5 --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-fdio-noha_daily.yaml @@ -0,0 +1,30 @@ +--- +# FDS suite + +schema: "yardstick:suite:0.1" + +name: "os-odl_l2-fdio-noha" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc001.yaml +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc006.yaml +- + file_name: opnfv_yardstick_tc007.yaml +- + file_name: opnfv_yardstick_tc008.yaml +- + file_name: opnfv_yardstick_tc009.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc020.yaml +- + file_name: opnfv_yardstick_tc021.yaml +- + file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc038.yaml diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-moon-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-moon-ha_daily.yaml index dadcb2f22..4a775b5bf 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-moon-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-moon-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-moon-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-ha_daily.yaml index 1de157a37..35358bcfe 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-nofeature-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-noha_daily.yaml index 1661e08f1..dc8b2efd0 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-nofeature-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-nofeature-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-ha_daily.yaml index 9e0e4186e..1899d407a 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-sfc-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-noha_daily.yaml index 1ebd73216..33f24e332 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l2-sfc-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l2-sfc-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-ha_daily.yaml index 4bcf81b45..97094bf32 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l3-nofeature-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-noha_daily.yaml index c50569b69..2796dca05 100644 --- a/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-odl_l3-nofeature-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-odl_l3-nofeature-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-onos-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-onos-nofeature-ha_daily.yaml index 48718abb7..777565ae5 100644 --- a/tests/opnfv/test_suites/opnfv_os-onos-nofeature-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-onos-nofeature-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-onos-nofeature-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-onos-nofeature-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-onos-nofeature-noha_daily.yaml index 0e9ff81d9..e6745613b 100644 --- a/tests/opnfv/test_suites/opnfv_os-onos-nofeature-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-onos-nofeature-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-onos-nofeature-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-onos-sfc-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-onos-sfc-ha_daily.yaml index bfb02cf48..aada4b450 100644 --- a/tests/opnfv/test_suites/opnfv_os-onos-sfc-ha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-onos-sfc-ha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-onos-sfc-ha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/opnfv/test_suites/opnfv_os-onos-sfc-noha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-onos-sfc-noha_daily.yaml index 2661e583e..a4e7c823f 100644 --- a/tests/opnfv/test_suites/opnfv_os-onos-sfc-noha_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_os-onos-sfc-noha_daily.yaml @@ -1,5 +1,5 @@ --- -# Huawei US bare daily task suite +# os-onos-sfc-noha daily task suite schema: "yardstick:suite:0.1" diff --git a/tests/unit/api/utils/test_common.py b/tests/unit/api/utils/test_common.py new file mode 100644 index 000000000..5d177409e --- /dev/null +++ b/tests/unit/api/utils/test_common.py @@ -0,0 +1,65 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest + +from api.utils import common + + +class TranslateToStrTestCase(unittest.TestCase): + + def test_translate_to_str_unicode(self): + input_str = u'hello' + output_str = common.translate_to_str(input_str) + + result = 'hello' + self.assertEqual(result, output_str) + + def test_translate_to_str_dict_list_unicode(self): + input_str = { + u'hello': {u'hello': [u'world']} + } + output_str = common.translate_to_str(input_str) + + result = { + 'hello': {'hello': ['world']} + } + self.assertEqual(result, output_str) + + +class GetCommandListTestCase(unittest.TestCase): + + def test_get_command_list_no_opts(self): + command_list = ['a'] + opts = {} + args = 'b' + output_list = common.get_command_list(command_list, opts, args) + + result_list = ['a', 'b'] + self.assertEqual(result_list, output_list) + + def test_get_command_list_with_opts_args(self): + command_list = ['a'] + opts = { + 'b': 'c', + 'task-args': 'd' + } + args = 'e' + + output_list = common.get_command_list(command_list, opts, args) + + result_list = ['a', 'e', '--b', '--task-args', 'd'] + self.assertEqual(result_list, output_list) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/api/utils/test_influx.py b/tests/unit/api/utils/test_influx.py new file mode 100644 index 000000000..0852da2dd --- /dev/null +++ b/tests/unit/api/utils/test_influx.py @@ -0,0 +1,82 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest +import mock +import uuid +import datetime + +from api.utils import influx + + +class GetDataDbClientTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx.ConfigParser') + def test_get_data_db_client_dispatcher_not_influxdb(self, mock_parser): + mock_parser.ConfigParser().get.return_value = 'file' + try: + influx.get_data_db_client() + except Exception as e: + self.assertIsInstance(e, RuntimeError) + + +class GetIpTestCase(unittest.TestCase): + + def test_get_url(self): + url = 'http://localhost:8086/hello' + output = influx._get_ip(url) + + result = 'localhost' + self.assertEqual(result, output) + + +class WriteDataTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx.get_data_db_client') + def test_write_data(self, mock_get_client): + measurement = 'tasklist' + field = {'status': 1} + timestamp = datetime.datetime.now() + tags = {'task_id': str(uuid.uuid4())} + + influx._write_data(measurement, field, timestamp, tags) + mock_get_client.assert_called_with() + + +class WriteDataTasklistTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx._write_data') + def test_write_data_tasklist(self, mock_write_data): + task_id = str(uuid.uuid4()) + timestamp = datetime.datetime.now() + status = 1 + influx.write_data_tasklist(task_id, timestamp, status) + + field = {'status': status, 'error': ''} + tags = {'task_id': task_id} + mock_write_data.assert_called_with('tasklist', field, timestamp, tags) + + +class QueryTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx.ConfigParser') + def test_query_dispatcher_not_influxdb(self, mock_parser): + mock_parser.ConfigParser().get.return_value = 'file' + try: + sql = 'select * form tasklist' + influx.query(sql) + except Exception as e: + self.assertIsInstance(e, RuntimeError) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/benchmark/scenarios/compute/test_computecapacity.py b/tests/unit/benchmark/scenarios/compute/test_computecapacity.py index 660bb3391..da06b5dbb 100644 --- a/tests/unit/benchmark/scenarios/compute/test_computecapacity.py +++ b/tests/unit/benchmark/scenarios/compute/test_computecapacity.py @@ -20,7 +20,7 @@ from yardstick.benchmark.scenarios.compute import computecapacity SAMPLE_OUTPUT = '{"Cpu_number": "2", "Core_number": "24",\ "Memory_size": "263753976 kB", "Thread_number": "48",\ - "Cache_size": "30720 KB"}' + "Cache_size": "30720 KB", "HT_Open": "0"}' @mock.patch('yardstick.benchmark.scenarios.compute.computecapacity.ssh') diff --git a/tests/unit/benchmark/scenarios/networking/test_networkcapacity.py b/tests/unit/benchmark/scenarios/networking/test_networkcapacity.py index e3a096446..e42832f1b 100644 --- a/tests/unit/benchmark/scenarios/networking/test_networkcapacity.py +++ b/tests/unit/benchmark/scenarios/networking/test_networkcapacity.py @@ -1,56 +1,56 @@ -#!/usr/bin/env python
-
-##############################################################################
-# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-# Unittest for yardstick.benchmark.scenarios.networking.networkcapacity.NetworkCapacity
-
-import mock
-import unittest
-import os
-import json
-
-from yardstick.benchmark.scenarios.networking import networkcapacity
-
-SAMPLE_OUTPUT = '{"Number of connections":"308","Number of frames received": "166503"}'
-
-@mock.patch('yardstick.benchmark.scenarios.networking.networkcapacity.ssh')
-class NetworkCapacityTestCase(unittest.TestCase):
-
- def setUp(self):
- self.ctx = {
- 'host': {
- 'ip': '172.16.0.137',
- 'user': 'cirros',
- 'password': "root"
- },
- }
-
- self.result = {}
-
- def test_capacity_successful_setup(self, mock_ssh):
- c = networkcapacity.NetworkCapacity({}, self.ctx)
- mock_ssh.SSH().execute.return_value = (0, '', '')
- c.setup()
- self.assertIsNotNone(c.client)
- self.assertTrue(c.setup_done)
-
- def test_capacity_successful(self, mock_ssh):
- c = networkcapacity.NetworkCapacity({}, self.ctx)
-
- mock_ssh.SSH().execute.return_value = (0, SAMPLE_OUTPUT, '')
- c.run(self.result)
- expected_result = json.loads(SAMPLE_OUTPUT)
- self.assertEqual(self.result, expected_result)
-
- def test_capacity_unsuccessful_script_error(self, mock_ssh):
- c = networkcapacity.NetworkCapacity({}, self.ctx)
-
- mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
- self.assertRaises(RuntimeError, c.run, self.result)
+#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.networking.networkcapacity.NetworkCapacity + +import mock +import unittest +import os +import json + +from yardstick.benchmark.scenarios.networking import networkcapacity + +SAMPLE_OUTPUT = '{"Number of connections":"308","Number of frames received": "166503"}' + +@mock.patch('yardstick.benchmark.scenarios.networking.networkcapacity.ssh') +class NetworkCapacityTestCase(unittest.TestCase): + + def setUp(self): + self.ctx = { + 'host': { + 'ip': '172.16.0.137', + 'user': 'cirros', + 'password': "root" + }, + } + + self.result = {} + + def test_capacity_successful_setup(self, mock_ssh): + c = networkcapacity.NetworkCapacity({}, self.ctx) + mock_ssh.SSH().execute.return_value = (0, '', '') + c.setup() + self.assertIsNotNone(c.client) + self.assertTrue(c.setup_done) + + def test_capacity_successful(self, mock_ssh): + c = networkcapacity.NetworkCapacity({}, self.ctx) + + mock_ssh.SSH().execute.return_value = (0, SAMPLE_OUTPUT, '') + c.run(self.result) + expected_result = json.loads(SAMPLE_OUTPUT) + self.assertEqual(self.result, expected_result) + + def test_capacity_unsuccessful_script_error(self, mock_ssh): + c = networkcapacity.NetworkCapacity({}, self.ctx) + + mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR') + self.assertRaises(RuntimeError, c.run, self.result) diff --git a/tests/unit/benchmark/scenarios/networking/test_ping6.py b/tests/unit/benchmark/scenarios/networking/test_ping6.py index b600e4103..0b8fba268 100644 --- a/tests/unit/benchmark/scenarios/networking/test_ping6.py +++ b/tests/unit/benchmark/scenarios/networking/test_ping6.py @@ -25,16 +25,33 @@ class PingTestCase(unittest.TestCase): 'host1': { 'ip': '172.16.0.137', 'user': 'cirros', + 'role': "Controller", 'key_filename': "mykey.key", 'password': "root" }, + 'host2': { + "ip": "172.16.0.138", + "key_filename": "/root/.ssh/id_rsa", + "role": "Compute", + "name": "node3.IPV6", + "user": "root" + }, } } + def test_get_controller_node(self): + args = { + 'options': {'host': 'host1','packetsize': 200, 'ping_count': 5}, + 'sla': {'max_rtt': 50} + } + p = ping6.Ping6(args, self.ctx) + controller_node = p._get_controller_node(['host1','host2']) + self.assertEqual(controller_node, 'host1') + @mock.patch('yardstick.benchmark.scenarios.networking.ping6.ssh') def test_ping_successful_setup(self, mock_ssh): args = { - 'options': {'host': 'host1','packetsize': 200}, + 'options': {'host': 'host1','packetsize': 200, 'ping_count': 5}, 'sla': {'max_rtt': 50} } p = ping6.Ping6(args, self.ctx) @@ -46,7 +63,7 @@ class PingTestCase(unittest.TestCase): @mock.patch('yardstick.benchmark.scenarios.networking.ping6.ssh') def test_ping_successful_no_sla(self, mock_ssh): args = { - 'options': {'host': 'host1','packetsize': 200}, + 'options': {'host': 'host1','packetsize': 200, 'ping_count': 5}, } result = {} @@ -61,7 +78,7 @@ class PingTestCase(unittest.TestCase): def test_ping_successful_sla(self, mock_ssh): args = { - 'options': {'host': 'host1','packetsize': 200}, + 'options': {'host': 'host1','packetsize': 200, 'ping_count': 5}, 'sla': {'max_rtt': 150} } result = {} @@ -76,7 +93,7 @@ class PingTestCase(unittest.TestCase): def test_ping_unsuccessful_sla(self, mock_ssh): args = { - 'options': {'host': 'host1','packetsize': 200}, + 'options': {'host': 'host1','packetsize': 200, 'ping_count': 5}, 'sla': {'max_rtt': 50} } result = {} @@ -90,7 +107,7 @@ class PingTestCase(unittest.TestCase): def test_ping_unsuccessful_script_error(self, mock_ssh): args = { - 'options': {'host': 'host1','packetsize': 200}, + 'options': {'host': 'host1','packetsize': 200, 'ping_count': 5}, 'sla': {'max_rtt': 150} } result = {} diff --git a/tests/unit/benchmark/scenarios/networking/test_vsperf.py b/tests/unit/benchmark/scenarios/networking/test_vsperf.py index cb5c09ab3..25d52212b 100644 --- a/tests/unit/benchmark/scenarios/networking/test_vsperf.py +++ b/tests/unit/benchmark/scenarios/networking/test_vsperf.py @@ -39,17 +39,17 @@ class VsperfTestCase(unittest.TestCase): } self.args = { 'options': { - 'testname': 'rfc2544_p2p_continuous', + 'testname': 'p2p_rfc2544_continuous', 'traffic_type': 'continuous', - 'pkt_sizes': '64', + 'frame_size': '64', 'bidirectional': 'True', 'iload': 100, - 'duration': 29, 'trafficgen_port1': 'eth1', 'trafficgen_port2': 'eth3', 'external_bridge': 'br-ex', - 'conf-file': 'vsperf-yardstick.conf', - 'setup-script': 'setup_yardstick.sh', + 'conf_file': 'vsperf-yardstick.conf', + 'setup_script': 'setup_yardstick.sh', + 'test_params': 'TRAFFICGEN_DURATION=30;', }, 'sla': { 'metrics': 'throughput_rx_fps', diff --git a/tests/unit/benchmark/scenarios/storage/test_storperf.py b/tests/unit/benchmark/scenarios/storage/test_storperf.py index d87ed733c..8fc97d2ed 100644 --- a/tests/unit/benchmark/scenarios/storage/test_storperf.py +++ b/tests/unit/benchmark/scenarios/storage/test_storperf.py @@ -43,7 +43,7 @@ def mocked_requests_job_get(*args, **kwargs): self.content = json_data self.status_code = status_code - return MockResponseJobGet('{"_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}', 200) + return MockResponseJobGet('{"status": "completed", "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}', 200) def mocked_requests_job_post(*args, **kwargs): @@ -152,7 +152,7 @@ class StorPerfTestCase(unittest.TestCase): s = storperf.StorPerf(args, self.ctx) s.setup_done = True - sample_output = '{"_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}' + sample_output = '{"status": "completed", "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}' expected_result = json.loads(sample_output) diff --git a/tests/unit/cmd/commands/test_env.py b/tests/unit/cmd/commands/test_env.py new file mode 100644 index 000000000..af1ab8030 --- /dev/null +++ b/tests/unit/cmd/commands/test_env.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest +import mock + +from yardstick.cmd.commands.env import EnvCommand + + +class EnvCommandTestCase(unittest.TestCase): + + @mock.patch('yardstick.cmd.commands.env.HttpClient') + def test_do_influxdb(self, mock_http_client): + env = EnvCommand() + env.do_influxdb({}) + self.assertTrue(mock_http_client().post.called) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/common/config_sample.yaml b/tests/unit/common/config_sample.yaml new file mode 100644 index 000000000..8caa19ec6 --- /dev/null +++ b/tests/unit/common/config_sample.yaml @@ -0,0 +1,2 @@ +releng: + dir: /home/opnfv/repos/releng diff --git a/tests/unit/common/test_httpClient.py b/tests/unit/common/test_httpClient.py new file mode 100644 index 000000000..b39dc2332 --- /dev/null +++ b/tests/unit/common/test_httpClient.py @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest +import mock +import json + +from yardstick.common import httpClient + + +class HttpClientTestCase(unittest.TestCase): + + @mock.patch('yardstick.common.httpClient.requests') + def test_post(self, mock_requests): + url = 'http://localhost:5000/hello' + data = {'hello': 'world'} + headers = {'Content-Type': 'application/json'} + httpClient.HttpClient().post(url, data) + mock_requests.post.assert_called_with(url, data=json.dumps(data), + headers=headers) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index 002d0494c..a64c1f1ab 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -83,6 +83,33 @@ class ImportModulesFromPackageTestCase(unittest.TestCase): mock_importutils.import_module.assert_called_with('bar.baz') +class GetParaFromYaml(unittest.TestCase): + + def test_get_para_from_yaml_file_not_exist(self): + file_path = '/etc/yardstick/hello.yaml' + args = 'hello.world' + para = utils.get_para_from_yaml(file_path, args) + self.assertIsNone(para) + + def test_get_para_from_yaml_para_not_found(self): + file_path = 'config_sample.yaml' + file_path = self._get_file_abspath(file_path) + args = 'releng.file' + self.assertIsNone(utils.get_para_from_yaml(file_path, args)) + + def test_get_para_from_yaml_para_exists(self): + file_path = 'config_sample.yaml' + file_path = self._get_file_abspath(file_path) + args = 'releng.dir' + para = '/home/opnfv/repos/releng' + self.assertEqual(para, utils.get_para_from_yaml(file_path, args)) + + def _get_file_abspath(self, filename): + curr_path = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(curr_path, filename) + return file_path + + def main(): unittest.main() diff --git a/tests/unit/dispatcher/test_influxdb_line_protocol.py b/tests/unit/dispatcher/test_influxdb_line_protocol.py index cb05bf4d2..42553c498 100644 --- a/tests/unit/dispatcher/test_influxdb_line_protocol.py +++ b/tests/unit/dispatcher/test_influxdb_line_protocol.py @@ -4,7 +4,7 @@ # influxdb-python/influxdb/tests/test_line_protocol.py import unittest -from yardstick.dispatcher.influxdb_line_protocol import make_lines +from third_party.influxdb.influxdb_line_protocol import make_lines class TestLineProtocol(unittest.TestCase): diff --git a/tests/unit/test_ssh.py b/tests/unit/test_ssh.py index a27052462..88638a0a8 100644 --- a/tests/unit/test_ssh.py +++ b/tests/unit/test_ssh.py @@ -17,7 +17,10 @@ # rally/tests/unit/common/test_sshutils.py import os +import socket import unittest +from cStringIO import StringIO + import mock from yardstick import ssh @@ -162,10 +165,10 @@ class SSHTestCase(unittest.TestCase): def test_send_command(self, mock_paramiko): paramiko_sshclient = self.test_client._get_client() with mock.patch.object(paramiko_sshclient, "exec_command") \ - as mock_paramiko_exec_command: + as mock_paramiko_exec_command: self.test_client.send_command('cmd') mock_paramiko_exec_command.assert_called_once_with('cmd', - get_pty=True) + get_pty=True) class SSHRunTestCase(unittest.TestCase): @@ -275,6 +278,23 @@ class SSHRunTestCase(unittest.TestCase): self.assertEqual(send_calls, self.fake_session.send.mock_calls) @mock.patch("yardstick.ssh.select") + def test_run_stdin_keep_open(self, mock_select): + """Test run method with stdin. + + Third send call was called with "e2" because only 3 bytes was sent + by second call. So remainig 2 bytes of "line2" was sent by third call. + """ + mock_select.select.return_value = ([], [], []) + self.fake_session.exit_status_ready.side_effect = [0, 0, 0, True] + self.fake_session.send_ready.return_value = True + self.fake_session.send.side_effect = len + fake_stdin = StringIO("line1\nline2\n") + self.test_client.run("cmd", stdin=fake_stdin, keep_stdin_open=True) + call = mock.call + send_calls = [call("line1\nline2\n")] + self.assertEqual(send_calls, self.fake_session.send.mock_calls) + + @mock.patch("yardstick.ssh.select") def test_run_select_error(self, mock_select): self.fake_session.exit_status_ready.return_value = False mock_select.select.return_value = ([], [], [True]) @@ -288,6 +308,61 @@ class SSHRunTestCase(unittest.TestCase): self.fake_session.exit_status_ready.return_value = False self.assertRaises(ssh.SSHTimeout, self.test_client.run, "cmd") + @mock.patch("yardstick.ssh.open", create=True) + def test__put_file_shell(self, mock_open): + self.test_client.run = mock.Mock() + self.test_client._put_file_shell("localfile", "remotefile", 0o42) + + self.test_client.run.assert_called_once_with( + 'cat > "remotefile"&& chmod -- 042 "remotefile"', + stdin=mock_open.return_value.__enter__.return_value) + + @mock.patch("yardstick.ssh.os.stat") + def test__put_file_sftp(self, mock_stat): + sftp = self.fake_client.open_sftp.return_value = mock.MagicMock() + sftp.__enter__.return_value = sftp + + mock_stat.return_value = os.stat_result([0o753] + [0] * 9) + + self.test_client._put_file_sftp("localfile", "remotefile") + + sftp.put.assert_called_once_with("localfile", "remotefile") + mock_stat.assert_called_once_with("localfile") + sftp.chmod.assert_called_once_with("remotefile", 0o753) + sftp.__exit__.assert_called_once_with(None, None, None) + + def test__put_file_sftp_mode(self): + sftp = self.fake_client.open_sftp.return_value = mock.MagicMock() + sftp.__enter__.return_value = sftp + + self.test_client._put_file_sftp("localfile", "remotefile", mode=0o753) + + sftp.put.assert_called_once_with("localfile", "remotefile") + sftp.chmod.assert_called_once_with("remotefile", 0o753) + sftp.__exit__.assert_called_once_with(None, None, None) + + def test_put_file_SSHException(self): + exc = ssh.paramiko.SSHException + self.test_client._put_file_sftp = mock.Mock(side_effect=exc()) + self.test_client._put_file_shell = mock.Mock() + + self.test_client.put_file("foo", "bar", 42) + self.test_client._put_file_sftp.assert_called_once_with("foo", "bar", + mode=42) + self.test_client._put_file_shell.assert_called_once_with("foo", "bar", + mode=42) + + def test_put_file_socket_error(self): + exc = socket.error + self.test_client._put_file_sftp = mock.Mock(side_effect=exc()) + self.test_client._put_file_shell = mock.Mock() + + self.test_client.put_file("foo", "bar", 42) + self.test_client._put_file_sftp.assert_called_once_with("foo", "bar", + mode=42) + self.test_client._put_file_shell.assert_called_once_with("foo", "bar", + mode=42) + def main(): unittest.main() diff --git a/third_party/__init__.py b/third_party/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/third_party/__init__.py diff --git a/third_party/influxdb/__init__.py b/third_party/influxdb/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/third_party/influxdb/__init__.py diff --git a/yardstick/dispatcher/influxdb_line_protocol.py b/third_party/influxdb/influxdb_line_protocol.py index eee982163..eee982163 100644 --- a/yardstick/dispatcher/influxdb_line_protocol.py +++ b/third_party/influxdb/influxdb_line_protocol.py diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh index 2e8399a9b..49c842c97 100755 --- a/tools/ubuntu-server-cloudimg-modify.sh +++ b/tools/ubuntu-server-cloudimg-modify.sh @@ -24,9 +24,13 @@ if [ $# -eq 1 ]; then fi # iperf3 only available for trusty in backports -grep trusty /etc/apt/sources.list && \ - echo "deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list - +if [ grep -q trusty /etc/apt/sources.list ]; then + if [ $YARD_IMG_ARCH = "arm64" ]; then + echo "deb [arch=arm64] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list + else + echo "deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list + fi +fi # Workaround for building on CentOS (apt-get is not working with http sources) # sed -i 's/http/ftp/' /etc/apt/sources.list @@ -41,8 +45,18 @@ password: RANDOM chpasswd: { expire: False } ssh_pwauth: True EOF - apt-get update +if [ $YARD_IMG_ARCH = "arm64" ]; then +apt-get install -y \ + linux-headers-$(echo $VIVID_KERNEL_VERSION | cut -d'-' -f3,4,5) \ + unzip +#resize root parition (/dev/vdb1) It is supposed to be default but the image is booted differently for arm64 +cat <<EOF >/etc/cloud/cloud.cfg.d/15_growpart.cfg +#cloud-config +bootcmd: + - [growpart, /dev/vdb, 1] +EOF +fi apt-get install -y \ fio \ git \ @@ -59,15 +73,37 @@ apt-get install -y \ stress \ sysstat -git clone https://github.com/kdlucas/byte-unixbench.git /opt/tempT +if [ $YARD_IMG_ARCH = "arm64" ]; then + wget https://github.com/kdlucas/byte-unixbench/archive/master.zip + unzip master.zip && rm master.zip + mkdir /opt/tempT + mv byte-unixbench-master/UnixBench /opt/tempT + sed -i -e 's/OPTON += -march=native -mtune=native/OPTON += -march=armv8-a -mtune=generic/g' \ + -e 's/OPTON += -march=native/OPTON += -march=armv8-a/g' /opt/tempT/UnixBench/Makefile +else + git clone https://github.com/kdlucas/byte-unixbench.git /opt/tempT +fi make --directory /opt/tempT/UnixBench/ -git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT/RAMspeed +if [ $YARD_IMG_ARCH = "arm64" ]; then + wget https://github.com/beefyamoeba5/ramspeed/archive/master.zip + unzip master.zip && rm master.zip + mkdir /opt/tempT/RAMspeed + mv ramspeed-master/* /opt/tempT/RAMspeed/ +else + git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT/RAMspeed +fi cd /opt/tempT/RAMspeed/ramspeed-2.6.0 mkdir temp bash build.sh -git clone https://github.com/beefyamoeba5/cachestat.git /opt/tempT/Cachestat +if [ $YARD_IMG_ARCH = "arm64" ]; then + wget https://github.com/beefyamoeba5/cachestat/archive/master.zip + unzip master.zip && rm master.zip + mv cachestat-master/cachestat /opt/tempT +else + git clone https://github.com/beefyamoeba5/cachestat.git /opt/tempT/Cachestat +fi # restore symlink ln -sf /run/resolvconf/resolv.conf /etc/resolv.conf diff --git a/tools/yardstick-img-lxd-modify b/tools/yardstick-img-lxd-modify new file mode 100755 index 000000000..4ca4eb489 --- /dev/null +++ b/tools/yardstick-img-lxd-modify @@ -0,0 +1,136 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# yardstick-img-lxd-modify - download and modify a Ubuntu cloud image +# +# The actual customization is done by a script passed with an absolute path as +# the only single argument. The command needs to be invoked as sudo +# +# Example invocation: +# yardstick-img-lxd-modify /home/yardstick/tools/ubuntu-server-cloudimg-modify.sh +# +# Warning: the script will create files by default in: +# /tmp/workspace/yardstick +# the files will be owned by root! +# +# TODO: image resize is needed if the base image is too small +# + +set -e +set -x + +die() { + echo "error: $1" >&2 + exit 1 +} + +test $# -eq 1 -o $# -eq 2 || die "no image specific script as argument" +test $(id -u) -eq 0 || die "should invoke using sudo" + +cmd=$1 +RELEASE=$2 +test -x $cmd +mountdir="/mnt/yardstick" +workspace=${WORKSPACE:-"/tmp/workspace/yardstick"} +host=${HOST:-"cloud-images.ubuntu.com"} +release=${RELEASE:-"xenial"} +image_path="${release}/current/${release}-server-cloudimg-amd64-root.tar.gz" +image_url=${IMAGE_URL:-"https://${host}/${image_path}"} +md5sums_path="${release}/current/MD5SUMS" +md5sums_url=${MD5SUMS_URL:-"https://${host}/${md5sums_path}"} + +imgfile="${workspace}/yardstick-image.tar.gz" +filename=$(basename $image_url) + +# download and checksum base image, conditionally if local copy is outdated +download() { + test -d $workspace || mkdir -p $workspace + cd $workspace + rm -f MD5SUMS # always download the checksum file to a detect stale image + wget $md5sums_url + test -e $filename || wget -nc --progress=dot:giga $image_url + grep $filename MD5SUMS | md5sum -c || + if [ $? -ne 0 ]; then + rm $filename + wget -nc --progress=dot:giga $image_url + grep $filename MD5SUMS | md5sum -c + fi + cd - +} + +# extract files +setup() { + mkdir -p $mountdir + + cp $cmd $mountdir/$(basename $cmd) + + cd $workspace + tar zxvf $filename -C $mountdir +} + +# modify image running a script using in a chrooted environment +modify() { + # resolv.conf does not exist in base image, pass nameserver value from host + nameserver_ip=$(grep -m 1 '^nameserver' \ + /etc/resolv.conf | awk '{ print $2 '}) + + # prevent init scripts from running during install + echo $'#!/bin/sh\nexit 101' >$mountdir/usr/sbin/policy-rc.d + chmod a+x $mountdir/usr/sbin/policy-rc.d + + chroot $mountdir /$(basename $cmd) $nameserver_ip + + rm -rf $mountdir/usr/sbin/policy-rc.d + + tar zcvf $(basename $imgfile) $mountdir/ +} + +# cleanup the image +cleanup() { + rm -rf $mountdir +} + +exitcode="" +error_trap() +{ + local rc=$? + + set +e + + if [ -z "$exitcode" ]; then + exitcode=$rc + fi + + dmesg -T | tail -50 + + cleanup + + echo "Image build failed with $exitcode" + + exit $exitcode +} + +main() { + cleanup + + trap "error_trap" EXIT SIGTERM + + download + setup + modify + + trap - EXIT SIGTERM + cleanup + + echo "the modified image is found here: $imgfile" +} + +main diff --git a/tools/yardstick-img-modify b/tools/yardstick-img-modify index 13d4360d9..c28e2ad59 100755 --- a/tools/yardstick-img-modify +++ b/tools/yardstick-img-modify @@ -32,22 +32,22 @@ die() { exit 1 } -test $# -eq 1 || die "no image specific script as argument" +test $# -eq 1 -o $# -eq 2 || die "no image specific script as argument" test $(id -u) -eq 0 || die "should invoke using sudo" cmd=$1 +RELEASE=$2 test -x $cmd mountdir="/mnt/yardstick" - workspace=${WORKSPACE:-"/tmp/workspace/yardstick"} host=${HOST:-"cloud-images.ubuntu.com"} -release=${RELEASE:-"trusty"} -image_path="${release}/current/${release}-server-cloudimg-amd64-disk1.img" +release=${RELEASE:-"xenial"} +image_path="${release}/current/${release}-server-cloudimg-${YARD_IMG_ARCH}-disk1.img" image_url=${IMAGE_URL:-"https://${host}/${image_path}"} md5sums_path="${release}/current/MD5SUMS" md5sums_url=${MD5SUMS_URL:-"https://${host}/${md5sums_path}"} -imgfile="${workspace}/yardstick-${release}-server.img" +imgfile="${workspace}/yardstick-image.img" raw_imgfile="${workspace}/yardstick-${release}-server.raw" filename=$(basename $image_url) @@ -64,30 +64,61 @@ download() { wget -nc --progress=dot:giga $image_url grep $filename MD5SUMS | md5sum -c fi + + for i in $(seq 0 9); do + [ -a /dev/loop$i ] || mknod -m 660 /dev/loop$i b 7 $i + done + + if [ $YARD_IMG_ARCH = "arm64" ]; then + cd /tmp + if [ ! -f /tmp/vivid-server-cloudimg-arm64-kernel-info.txt ]; then + wget http://cloud-images.ubuntu.com/vivid/current/vivid-server-cloudimg-arm64-kernel-info.txt + fi + export VIVID_KERNEL_VERSION=$(cut -d$'\t' -f4 vivid-server-cloudimg-arm64-kernel-info.txt) + mkdir -p /tmp/vivid-modules + if [ ! -f "/tmp/vivid-server-cloudimg-arm64.tar.gz" ]; then + wget $VIVID_IMG_URL + fi + if [ ! -f "/tmp/vivid-server-cloudimg-arm64.img" ]; then + tar zxvf vivid-server-cloudimg-arm64.tar.gz vivid-server-cloudimg-arm64.img + fi + mkdir -p /mnt/vivid + mount /tmp/vivid-server-cloudimg-arm64.img /mnt/vivid + cp -r /mnt/vivid/lib/modules/$(echo $VIVID_KERNEL_VERSION | cut -d'-' -f3,4,5) /tmp/vivid-modules + umount /mnt/vivid + rm /tmp/vivid-server-cloudimg-arm64.img + cd $workspace + fi qemu-img convert $filename $raw_imgfile cd - } # mount image setup() { + # qemu-img resize $raw_imgfile +5GB + if [ $YARD_IMG_ARCH = "arm64" ]; then + echo -e "d\nn\np\n1\n\n\nw" | fdisk $raw_imgfile + fi mkdir -p $mountdir - for i in $(seq 0 9); do - [ -a /dev/loop$i ] || mknod -m 660 /dev/loop$i b 7 $i - done - loopdevice=$(kpartx -l $raw_imgfile | head -1 | cut -f1 -d ' ') kpartx -av $raw_imgfile - + if [ $YARD_IMG_ARCH = "arm64" ]; then + e2fsck -p -f /dev/mapper/$loopdevice + resize2fs /dev/mapper/$loopdevice + fi # for trouble shooting sleep 2 dmsetup ls fdisk -l /dev/${loopdevice:0:5} || true - mount /dev/mapper/$loopdevice $mountdir mount -t proc none $mountdir/proc + if [ $YARD_IMG_ARCH = "arm64" ]; then + cp -r /tmp/vivid-modules/$(echo $VIVID_KERNEL_VERSION | cut -d'-' -f3,4,5) "$mountdir/lib/modules" + cp $(which "qemu-aarch64-static") "$mountdir/usr/bin" + fi cp $cmd $mountdir/$(basename $cmd) } @@ -120,6 +151,7 @@ cleanup() { # designed to be idempotent mount | grep $mountdir/proc && umount $mountdir/proc mount | grep $mountdir && umount $mountdir + mount | grep "/mnt/vivid" && umount "/mnt/vivid" if [ -f $raw_imgfile ]; then kpartx -dv $raw_imgfile || true fi diff --git a/yardstick/__init__.py b/yardstick/__init__.py index c31948d43..5c279c800 100644 --- a/yardstick/__init__.py +++ b/yardstick/__init__.py @@ -8,18 +8,40 @@ ############################################################################## import logging -import logging.config -import sys import os +import sys + import yardstick.vTC.apexlake as apexlake # Hack to be able to run apexlake unit tests # without having to install apexlake. sys.path.append(os.path.dirname(apexlake.__file__)) -logging.basicConfig( - level=logging.WARNING, - format='[%(asctime)s] %(name)-20s %(filename)s:%(lineno)d ' - '%(levelname)s %(message)s', # noqa - datefmt='%m/%d/%y %H:%M:%S') -logging.getLogger(__name__).setLevel(logging.INFO) +LOG_FILE = '/tmp/yardstick.log' +LOG_FORMATTER = ('%(asctime)s ' + '%(name)s %(filename)s:%(lineno)d ' + '%(levelname)s %(message)s') + +_LOG_FORMATTER = logging.Formatter(LOG_FORMATTER) +_LOG_STREAM_HDLR = logging.StreamHandler() +_LOG_FILE_HDLR = logging.FileHandler(LOG_FILE) + +LOG = logging.getLogger(__name__) + + +def _init_logging(): + + _LOG_STREAM_HDLR.setFormatter(_LOG_FORMATTER) + + # don't append to log file, clobber + _LOG_FILE_HDLR.setFormatter(_LOG_FORMATTER) + + del logging.root.handlers[:] + logging.root.addHandler(_LOG_STREAM_HDLR) + logging.root.addHandler(_LOG_FILE_HDLR) + logging.debug("logging.root.handlers = %s", logging.root.handlers) + + if os.environ.get('CI_DEBUG', '').lower() in {'1', 1, 'y', "yes", "true"}: + LOG.setLevel(logging.DEBUG) + else: + LOG.setLevel(logging.INFO) diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index 8c514d250..fcbe825d6 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -7,8 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import os import sys import pkg_resources +import paramiko from yardstick.benchmark.contexts.base import Context from yardstick.benchmark.contexts.model import Server @@ -16,6 +18,7 @@ from yardstick.benchmark.contexts.model import PlacementGroup from yardstick.benchmark.contexts.model import Network from yardstick.benchmark.contexts.model import update_scheduler_hints from yardstick.orchestrator.heat import HeatTemplate +from yardstick.definitions import YARDSTICK_ROOT_PATH class HeatContext(Context): @@ -37,6 +40,8 @@ class HeatContext(Context): self._user = None self.template_file = None self.heat_parameters = None + self.key_filename = YARDSTICK_ROOT_PATH + \ + 'yardstick/resources/files/yardstick_key' super(self.__class__, self).__init__() def init(self, attrs): @@ -74,6 +79,17 @@ class HeatContext(Context): self.servers.append(server) self._server_map[server.dn] = server + print "Generating RSA host key ..." + rsa_key = paramiko.RSAKey.generate(bits=2048, progress_func=None) + print "Writing yardstick_key ..." + rsa_key.write_private_key_file(self.key_filename) + print "Writing yardstick_key.pub ..." + open(self.key_filename + ".pub", "w").write("%s %s\n" % + (rsa_key.get_name(), + rsa_key.get_base64())) + del rsa_key + print "... done!" + @property def image(self): '''returns application's default image name''' @@ -214,6 +230,13 @@ class HeatContext(Context): self.stack = None print "Context '%s' undeployed" % self.name + if os.path.exists(self.key_filename): + try: + os.remove(self.key_filename) + os.remove(self.key_filename + ".pub") + except OSError, e: + print ("Error: %s - %s." % (e.key_filename, e.strerror)) + def _get_server(self, attr_name): '''lookup server info by name from context attr_name: either a name for a server created by yardstick or a dict diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py index c3d652119..78bce8259 100644 --- a/yardstick/benchmark/contexts/node.py +++ b/yardstick/benchmark/contexts/node.py @@ -8,10 +8,12 @@ ############################################################################## import sys +import os import yaml import logging from yardstick.benchmark.contexts.base import Context +from yardstick.definitions import YARDSTICK_ROOT_PATH LOG = logging.getLogger(__name__) @@ -33,7 +35,9 @@ class NodeContext(Context): def init(self, attrs): '''initializes itself from the supplied arguments''' self.name = attrs["name"] - self.file_path = attrs.get("file", "/etc/yardstick/nodes/pod.yaml") + self.file_path = attrs.get("file", "pod.yaml") + if not os.path.exists(self.file_path): + self.file_path = os.path.join(YARDSTICK_ROOT_PATH, self.file_path) LOG.info("Parsing pod file: %s", self.file_path) @@ -79,7 +83,7 @@ class NodeContext(Context): return None elif len(nodes) > 1: LOG.error("Duplicate nodes!!!") - LOG.error("Nodes: %r" % nodes) + LOG.error("Nodes: %r", nodes) sys.exit(-1) # A clone is created in order to avoid affecting the diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py index 74a236f44..69ea915a1 100755 --- a/yardstick/benchmark/runners/arithmetic.py +++ b/yardstick/benchmark/runners/arithmetic.py @@ -93,7 +93,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if aborted.is_set(): break - LOG.debug("runner=%(runner)s seq=%(sequence)s START" % + LOG.debug("runner=%(runner)s seq=%(sequence)s START", {"runner": runner_cfg["runner_id"], "sequence": sequence}) for i, value in enumerate(comb_values): @@ -109,7 +109,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if sla_action == "assert": raise elif sla_action == "monitor": - LOG.warning("SLA validation failed: %s" % assertion.args) + LOG.warning("SLA validation failed: %s", assertion.args) errors = assertion.args except Exception as e: errors = traceback.format_exc() @@ -129,7 +129,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, queue.put(record) - LOG.debug("runner=%(runner)s seq=%(sequence)s END" % + LOG.debug("runner=%(runner)s seq=%(sequence)s END", {"runner": runner_cfg["runner_id"], "sequence": sequence}) sequence += 1 diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py index 23749924f..8f3f75fa1 100755 --- a/yardstick/benchmark/runners/base.py +++ b/yardstick/benchmark/runners/base.py @@ -63,7 +63,7 @@ def _execute_shell_command(command): except Exception: exitcode = -1 output = traceback.format_exc() - log.error("exec command '%s' error:\n " % command) + log.error("exec command '%s' error:\n ", command) log.error(traceback.format_exc()) return exitcode, output @@ -76,10 +76,10 @@ def _single_action(seconds, command, queue): log.debug("single action: executing command: '%s'", command) ret_code, data = _execute_shell_command(command) if ret_code < 0: - log.error("single action error! command:%s" % command) + log.error("single action error! command:%s", command) queue.put({'single-action-data': data}) return - log.debug("single action data: \n%s" % data) + log.debug("single action data: \n%s", data) queue.put({'single-action-data': data}) @@ -96,7 +96,7 @@ def _periodic_action(interval, command, queue): log.error("periodic action error! command:%s", command) queue.put({'periodic-action-data': data}) break - log.debug("periodic action data: \n%s" % data) + log.debug("periodic action data: \n%s", data) queue.put({'periodic-action-data': data}) @@ -127,7 +127,7 @@ class Runner(object): """ # if there is no runner, start the output serializer subprocess if len(Runner.runners) == 0: - log.debug("Starting dump process file '%s'" % + log.debug("Starting dump process file '%s'", config["output_filename"]) Runner.queue = multiprocessing.Queue() Runner.dump_process = multiprocessing.Process( @@ -196,13 +196,13 @@ class Runner(object): '''run a potentially configured post-stop action''' if "post-stop-action" in self.config: command = self.config["post-stop-action"]["command"] - log.debug("post stop action: command: '%s'" % command) + log.debug("post stop action: command: '%s'", command) ret_code, data = _execute_shell_command(command) if ret_code < 0: log.error("post action error! command:%s", command) self.result_queue.put({'post-stop-action-data': data}) return - log.debug("post-stop data: \n%s" % data) + log.debug("post-stop data: \n%s", data) self.result_queue.put({'post-stop-action-data': data}) def run(self, scenario_cfg, context_cfg): @@ -219,13 +219,13 @@ class Runner(object): # run a potentially configured pre-start action if "pre-start-action" in self.config: command = self.config["pre-start-action"]["command"] - log.debug("pre start action: command: '%s'" % command) + log.debug("pre start action: command: '%s'", command) ret_code, data = _execute_shell_command(command) if ret_code < 0: log.error("pre-start action error! command:%s", command) self.result_queue.put({'pre-start-action-data': data}) return - log.debug("pre-start data: \n%s" % data) + log.debug("pre-start data: \n%s", data) self.result_queue.put({'pre-start-action-data': data}) if "single-shot-action" in self.config: diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py index 1f51f513f..1412c0caa 100644 --- a/yardstick/benchmark/runners/duration.py +++ b/yardstick/benchmark/runners/duration.py @@ -58,7 +58,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, start = time.time() while True: - LOG.debug("runner=%(runner)s seq=%(sequence)s START" % + LOG.debug("runner=%(runner)s seq=%(sequence)s START", {"runner": runner_cfg["runner_id"], "sequence": sequence}) data = {} @@ -71,7 +71,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if sla_action == "assert": raise elif sla_action == "monitor": - LOG.warning("SLA validation failed: %s" % assertion.args) + LOG.warning("SLA validation failed: %s", assertion.args) errors = assertion.args except Exception as e: errors = traceback.format_exc() @@ -91,7 +91,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, queue.put(record) - LOG.debug("runner=%(runner)s seq=%(sequence)s END" % + LOG.debug("runner=%(runner)s seq=%(sequence)s END", {"runner": runner_cfg["runner_id"], "sequence": sequence}) sequence += 1 diff --git a/yardstick/benchmark/runners/iteration.py b/yardstick/benchmark/runners/iteration.py index b23b32b08..3a839b65f 100644 --- a/yardstick/benchmark/runners/iteration.py +++ b/yardstick/benchmark/runners/iteration.py @@ -60,7 +60,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if "run" in run_step: while True: - LOG.debug("runner=%(runner)s seq=%(sequence)s START" % + LOG.debug("runner=%(runner)s seq=%(sequence)s START", {"runner": runner_cfg["runner_id"], "sequence": sequence}) @@ -74,7 +74,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if sla_action == "assert": raise elif sla_action == "monitor": - LOG.warning("SLA validation failed: %s" % assertion.args) + LOG.warning("SLA validation failed: %s", assertion.args) errors = assertion.args except Exception as e: errors = traceback.format_exc() @@ -94,7 +94,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, queue.put(record) - LOG.debug("runner=%(runner)s seq=%(sequence)s END" % + LOG.debug("runner=%(runner)s seq=%(sequence)s END", {"runner": runner_cfg["runner_id"], "sequence": sequence}) diff --git a/yardstick/benchmark/runners/sequence.py b/yardstick/benchmark/runners/sequence.py index fe53412ca..3b06e2a36 100644 --- a/yardstick/benchmark/runners/sequence.py +++ b/yardstick/benchmark/runners/sequence.py @@ -67,7 +67,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, for value in sequence_values: options[arg_name] = value - LOG.debug("runner=%(runner)s seq=%(sequence)s START" % + LOG.debug("runner=%(runner)s seq=%(sequence)s START", {"runner": runner_cfg["runner_id"], "sequence": sequence}) data = {} @@ -80,7 +80,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if sla_action == "assert": raise elif sla_action == "monitor": - LOG.warning("SLA validation failed: %s" % assertion.args) + LOG.warning("SLA validation failed: %s", assertion.args) errors = assertion.args except Exception as e: errors = traceback.format_exc() @@ -100,7 +100,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, queue.put(record) - LOG.debug("runner=%(runner)s seq=%(sequence)s END" % + LOG.debug("runner=%(runner)s seq=%(sequence)s END", {"runner": runner_cfg["runner_id"], "sequence": sequence}) sequence += 1 diff --git a/yardstick/benchmark/scenarios/availability/actionrollbackers.py b/yardstick/benchmark/scenarios/availability/actionrollbackers.py index 4b732a10c..38f57d476 100644 --- a/yardstick/benchmark/scenarios/availability/actionrollbackers.py +++ b/yardstick/benchmark/scenarios/availability/actionrollbackers.py @@ -28,8 +28,8 @@ class AttackerRollbacker(ActionRollbacker): def rollback(self): LOG.debug( - "\033[93m recovering attacker %s \033[0m" - % (self.underlyingAttacker.key)) + "\033[93m recovering attacker %s \033[0m", + self.underlyingAttacker.key) self.underlyingAttacker.recover() @@ -40,6 +40,6 @@ class OperationRollbacker(ActionRollbacker): def rollback(self): LOG.debug( - "\033[93m rollback operation %s \033[0m" - % (self.underlyingOperation.key)) + "\033[93m rollback operation %s \033[0m", + self.underlyingOperation.key) self.underlyingOperation.rollback() diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_baremetal.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_baremetal.py index b35869d07..e88fed636 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/attacker_baremetal.py +++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_baremetal.py @@ -24,24 +24,25 @@ def _execute_shell_command(command, stdin=None): except Exception: exitcode = -1 output = traceback.format_exc() - LOG.error("exec command '%s' error:\n " % command) + LOG.error("exec command '%s' error:\n ", command) LOG.error(traceback.format_exc()) return exitcode, output class BaremetalAttacker(BaseAttacker): - __attacker_type__ = 'bare-metal-down' def setup(self): - LOG.debug("config:%s context:%s" % (self._config, self._context)) + LOG.debug("config:%s context:%s", self._config, self._context) host = self._context.get(self._config['host'], None) ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") self.host_ip = ip @@ -60,14 +61,15 @@ class BaremetalAttacker(BaseAttacker): self.setup_done = True def check(self): - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s {0} -W 10".format(self.host_ip), - stdin=open(self.check_script, "r")) + with open(self.check_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0} -W 10".format(self.host_ip), + stdin=stdin_file) - LOG.debug("check ret: %s out:%s err:%s" % - (exit_status, stdout, stderr)) + LOG.debug("check ret: %s out:%s err:%s", + exit_status, stdout, stderr) if not stdout or "running" not in stdout: - LOG.info("the host (ipmi_ip:%s) is not running!" % self.ipmi_ip) + LOG.info("the host (ipmi_ip:%s) is not running!", self.ipmi_ip) return False return True @@ -75,8 +77,8 @@ class BaremetalAttacker(BaseAttacker): def inject_fault(self): exit_status, stdout, stderr = self.connection.execute( "shutdown -h now") - LOG.debug("inject fault ret: %s out:%s err:%s" % - (exit_status, stdout, stderr)) + LOG.debug("inject fault ret: %s out:%s err:%s", + exit_status, stdout, stderr) if not exit_status: LOG.info("inject fault success") @@ -87,18 +89,21 @@ class BaremetalAttacker(BaseAttacker): host = self._context.get(jump_host_name, None) ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) pwd = host.get("pwd", None) - LOG.debug("jump_host ip:%s user:%s" % (ip, user)) - self.jump_connection = ssh.SSH(user, ip, password=pwd) + LOG.debug("jump_host ip:%s user:%s", ip, user) + self.jump_connection = ssh.SSH(user, ip, password=pwd, + port=ssh_port) self.jump_connection.wait(timeout=600) LOG.debug("ssh jump host success!") if self.jump_connection is not None: - exit_status, stdout, stderr = self.jump_connection.execute( - "/bin/bash -s {0} {1} {2} {3}".format( - self.ipmi_ip, self.ipmi_user, self.ipmi_pwd, "on"), - stdin=open(self.recovery_script, "r")) + with open(self.recovery_script, "r") as stdin_file: + exit_status, stdout, stderr = self.jump_connection.execute( + "/bin/bash -s {0} {1} {2} {3}".format( + self.ipmi_ip, self.ipmi_user, self.ipmi_pwd, "on"), + stdin=stdin_file) else: exit_status, stdout = _execute_shell_command( "/bin/bash -s {0} {1} {2} {3}".format( diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py index 816e7e37d..595067a95 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py +++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py @@ -20,13 +20,15 @@ class GeneralAttacker(BaseAttacker): __attacker_type__ = 'general-attacker' def setup(self): - LOG.debug("config:%s context:%s" % (self._config, self._context)) + LOG.debug("config:%s context:%s", self._config, self._context) host = self._context.get(self._config['host'], None) ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") @@ -63,13 +65,15 @@ class GeneralAttacker(BaseAttacker): if "action_parameter" in self._config: LOG.debug("the shell command is: {0}".format(self.action_param)) - exit_status, stdout, stderr = self.connection.execute( - self.action_param, - stdin=open(self.inject_script, "r")) + with open(self.inject_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + self.action_param, + stdin=stdin_file) else: - exit_status, stdout, stderr = self.connection.execute( - "/bin/bash -s ", - stdin=open(self.inject_script, "r")) + with open(self.inject_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/bash -s ", + stdin=stdin_file) LOG.debug("the inject_fault's exit status is: {0}".format(exit_status)) if exit_status == 0: @@ -77,16 +81,18 @@ class GeneralAttacker(BaseAttacker): .format(stdout)) else: LOG.error( - "the inject_fault's error, stdout:%s, stderr:%s" % - (stdout, stderr)) + "the inject_fault's error, stdout:%s, stderr:%s", + stdout, stderr) def recover(self): if "rollback_parameter" in self._config: LOG.debug("the shell command is: {0}".format(self.rollback_param)) - exit_status, stdout, stderr = self.connection.execute( - self.rollback_param, - stdin=open(self.recovery_script, "r")) + with open(self.recovery_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + self.rollback_param, + stdin=stdin_file) else: - exit_status, stdout, stderr = self.connection.execute( - "/bin/bash -s ", - stdin=open(self.recovery_script, "r")) + with open(self.recovery_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/bash -s ", + stdin=stdin_file) diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py index 5118ad628..1d190a160 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py +++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py @@ -19,13 +19,15 @@ class ProcessAttacker(BaseAttacker): __attacker_type__ = 'kill-process' def setup(self): - LOG.debug("config:%s context:%s" % (self._config, self._context)) + LOG.debug("config:%s context:%s", self._config, self._context) host = self._context.get(self._config['host'], None) ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") @@ -43,25 +45,28 @@ class ProcessAttacker(BaseAttacker): self.setup_done = True def check(self): - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s {0}".format(self.service_name), - stdin=open(self.check_script, "r")) + with open(self.check_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.service_name), + stdin=stdin_file) if stdout and "running" in stdout: LOG.info("check the envrioment success!") return True else: LOG.error( - "the host envrioment is error, stdout:%s, stderr:%s" % - (stdout, stderr)) + "the host envrioment is error, stdout:%s, stderr:%s", + stdout, stderr) return False def inject_fault(self): - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s {0}".format(self.service_name), - stdin=open(self.inject_script, "r")) + with open(self.inject_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.service_name), + stdin=stdin_file) def recover(self): - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s {0} ".format(self.service_name), - stdin=open(self.recovery_script, "r")) + with open(self.recovery_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0} ".format(self.service_name), + stdin=stdin_file) diff --git a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py index 78276efa2..f96e57728 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py +++ b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py @@ -26,7 +26,7 @@ class AttackerMgr(object): self._attacker_list = [] def init_attackers(self, attacker_cfgs, context): - LOG.debug("attackerMgr confg: %s" % attacker_cfgs) + LOG.debug("attackerMgr confg: %s", attacker_cfgs) for cfg in attacker_cfgs: attacker_cls = BaseAttacker.get_attacker_cls(cfg) diff --git a/yardstick/benchmark/scenarios/availability/director.py b/yardstick/benchmark/scenarios/availability/director.py index 267933dd0..104c68380 100644 --- a/yardstick/benchmark/scenarios/availability/director.py +++ b/yardstick/benchmark/scenarios/availability/director.py @@ -63,7 +63,7 @@ class Director(object): def createActionPlayer(self, type, key): LOG.debug( - "the type of current action is %s, the key is %s" % (type, key)) + "the type of current action is %s, the key is %s", type, key) if type == ActionType.ATTACKER: return actionplayers.AttackerPlayer(self.attackerMgr[key]) if type == ActionType.MONITOR: @@ -77,13 +77,13 @@ class Director(object): def createActionRollbacker(self, type, key): LOG.debug( - "the type of current action is %s, the key is %s" % (type, key)) + "the type of current action is %s, the key is %s", type, key) if type == ActionType.ATTACKER: return actionrollbackers.AttackerRollbacker(self.attackerMgr[key]) if type == ActionType.OPERATION: return actionrollbackers.OperationRollbacker( self.operationMgr[key]) - LOG.debug("no rollbacker created for %s" % (key)) + LOG.debug("no rollbacker created for %s", key) def verify(self): result = True diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py index d26c99c75..38d1c4e5c 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py +++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py @@ -27,7 +27,7 @@ class MonitorMgr(object): self._monitor_list = [] def init_monitors(self, monitor_cfgs, context): - LOG.debug("monitorMgr config: %s" % monitor_cfgs) + LOG.debug("monitorMgr config: %s", monitor_cfgs) for monitor_cfg in monitor_cfgs: monitor_type = monitor_cfg["monitor_type"] @@ -87,7 +87,7 @@ class BaseMonitor(multiprocessing.Process): return os.path.join(base_path, path) def run(self): - LOG.debug("config:%s context:%s" % (self._config, self._context)) + LOG.debug("config:%s context:%s", self._config, self._context) self.setup() monitor_time = self._config.get("monitor_time", 0) @@ -140,7 +140,7 @@ class BaseMonitor(multiprocessing.Process): def wait_monitor(self): self.join() self._result = self._queue.get() - LOG.debug("the monitor result:%s" % self._result) + LOG.debug("the monitor result:%s", self._result) def setup(self): # pragma: no cover pass diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py index c285024e1..cd33e6188 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py @@ -24,7 +24,7 @@ def _execute_shell_command(command): except Exception: exitcode = -1 output = traceback.format_exc() - LOG.error("exec command '%s' error:\n " % command) + LOG.error("exec command '%s' error:\n ", command) LOG.error(traceback.format_exc()) return exitcode, output @@ -42,9 +42,11 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor): host = self._context[node_name] ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") @@ -56,12 +58,13 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor): def monitor_func(self): exit_status = 0 if self.connection: - exit_status, stdout, stderr = self.connection.execute( - "/bin/bash -s '{0}'".format(self.cmd), - stdin=open(self.check_script, "r")) + with open(self.check_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/bash -s '{0}'".format(self.cmd), + stdin=stdin_file) - LOG.debug("the ret stats: %s stdout: %s stderr: %s" % - (exit_status, stdout, stderr)) + LOG.debug("the ret stats: %s stdout: %s stderr: %s", + exit_status, stdout, stderr) else: exit_status, stdout = _execute_shell_command(self.cmd) if exit_status: @@ -70,10 +73,10 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor): def verify_SLA(self): outage_time = self._result.get('outage_time', None) - LOG.debug("the _result:%s" % self._result) + LOG.debug("the _result:%s", self._result) max_outage_time = self._config["sla"]["max_outage_time"] if outage_time > max_outage_time: - LOG.info("SLA failure: %f > %f" % (outage_time, max_outage_time)) + LOG.info("SLA failure: %f > %f", outage_time, max_outage_time) return False else: LOG.info("the sla is passed") diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py index 61efc0520..461a2ded5 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py @@ -25,6 +25,7 @@ class GeneralMonitor(basemonitor.BaseMonitor): host = self._context[self._config["host"]] ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") self.key = self._config["key"] self.monitor_key = self._config["monitor_key"] @@ -40,33 +41,36 @@ class GeneralMonitor(basemonitor.BaseMonitor): self.monitor_key) self.monitor_script = self.get_script_fullpath( self.monitor_cfg['monitor_script']) - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") def monitor_func(self): if "parameter" in self._config: - exit_status, stdout, stderr = self.connection.execute( - self.cmd_param, - stdin=open(self.monitor_script, "r")) + with open(self.monitor_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + self.cmd_param, + stdin=stdin_file) else: - exit_status, stdout, stderr = self.connection.execute( - "/bin/bash -s ", - stdin=open(self.monitor_script, "r")) + with open(self.monitor_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/bash -s ", + stdin=stdin_file) if exit_status: return False return True def verify_SLA(self): - LOG.debug("the _result:%s" % self._result) + LOG.debug("the _result:%s", self._result) outage_time = self._result.get('outage_time', None) max_outage_time = self._config["sla"]["max_outage_time"] if outage_time is None: LOG.error("There is no outage_time in monitor result.") return False if outage_time > max_outage_time: - LOG.error("SLA failure: %f > %f" % (outage_time, max_outage_time)) + LOG.error("SLA failure: %f > %f", outage_time, max_outage_time) return False else: return True diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py index 53a6d8e4d..5f492ad69 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py @@ -23,9 +23,11 @@ class MonitorProcess(basemonitor.BaseMonitor): host = self._context[self._config["host"]] ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") self.check_script = self.get_script_fullpath( @@ -33,21 +35,22 @@ class MonitorProcess(basemonitor.BaseMonitor): self.process_name = self._config["process_name"] def monitor_func(self): - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s {0}".format(self.process_name), - stdin=open(self.check_script, "r")) + with open(self.check_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.process_name), + stdin=stdin_file) if not stdout or int(stdout) <= 0: - LOG.info("the process (%s) is not running!" % self.process_name) + LOG.info("the process (%s) is not running!", self.process_name) return False return True def verify_SLA(self): - LOG.debug("the _result:%s" % self._result) + LOG.debug("the _result:%s", self._result) outage_time = self._result.get('outage_time', None) max_outage_time = self._config["sla"]["max_recover_time"] if outage_time > max_outage_time: - LOG.error("SLA failure: %f > %f" % (outage_time, max_outage_time)) + LOG.error("SLA failure: %f > %f", outage_time, max_outage_time) return False else: return True diff --git a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py index e776e87ae..80efd1b02 100644 --- a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py +++ b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py @@ -26,7 +26,7 @@ class OperationMgr(object): self._operation_list = [] def init_operations(self, operation_cfgs, context): - LOG.debug("operationMgr confg: %s" % operation_cfgs) + LOG.debug("operationMgr confg: %s", operation_cfgs) for cfg in operation_cfgs: operation_type = cfg['operation_type'] operation_cls = BaseOperation.get_operation_cls(operation_type) diff --git a/yardstick/benchmark/scenarios/availability/operation/operation_general.py b/yardstick/benchmark/scenarios/availability/operation/operation_general.py index e43f6e1d5..c82df836d 100644 --- a/yardstick/benchmark/scenarios/availability/operation/operation_general.py +++ b/yardstick/benchmark/scenarios/availability/operation/operation_general.py @@ -19,13 +19,15 @@ class GeneralOperaion(BaseOperation): __operation__type__ = "general-operation" def setup(self): - LOG.debug("config:%s context:%s" % (self._config, self._context)) + LOG.debug("config:%s context:%s", self._config, self._context) host = self._context.get(self._config['host'], None) ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") @@ -53,27 +55,31 @@ class GeneralOperaion(BaseOperation): def run(self): if "action_parameter" in self._config: - exit_status, stdout, stderr = self.connection.execute( - self.action_param, - stdin=open(self.action_script, "r")) + with open(self.action_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + self.action_param, + stdin=stdin_file) else: - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s ", - stdin=open(self.action_script, "r")) + with open(self.action_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s ", + stdin=stdin_file) if exit_status == 0: LOG.debug("success,the operation's output is: {0}".format(stdout)) else: LOG.error( - "the operation's error, stdout:%s, stderr:%s" % - (stdout, stderr)) + "the operation's error, stdout:%s, stderr:%s", + stdout, stderr) def rollback(self): if "rollback_parameter" in self._config: - exit_status, stdout, stderr = self.connection.execute( - self.rollback_param, - stdin=open(self.rollback_script, "r")) + with open(self.rollback_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + self.rollback_param, + stdin=stdin_file) else: - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s ", - stdin=open(self.rollback_script, "r")) + with open(self.rollback_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s ", + stdin=stdin_file) diff --git a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py index 1bdb9f2c2..a24f26e81 100644 --- a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py +++ b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py @@ -26,7 +26,7 @@ class ResultCheckerMgr(object): self._result_checker_list = [] def init_ResultChecker(self, resultchecker_cfgs, context): - LOG.debug("resultcheckerMgr confg: %s" % resultchecker_cfgs) + LOG.debug("resultcheckerMgr confg: %s", resultchecker_cfgs) for cfg in resultchecker_cfgs: resultchecker_type = cfg['checker_type'] diff --git a/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py b/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py index 681fbf63f..275aff076 100644 --- a/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py +++ b/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py @@ -17,17 +17,18 @@ LOG = logging.getLogger(__name__) class GeneralResultChecker(BaseResultChecker): - __result_checker__type__ = "general-result-checker" def setup(self): - LOG.debug("config:%s context:%s" % (self._config, self._context)) + LOG.debug("config:%s context:%s", self._config, self._context) host = self._context.get(self._config['host'], None) ip = host.get("ip", None) user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get("key_filename", "~/.ssh/id_rsa") - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) LOG.debug("ssh host success!") @@ -52,21 +53,23 @@ class GeneralResultChecker(BaseResultChecker): def verify(self): if "parameter" in self._config: - exit_status, stdout, stderr = self.connection.execute( - self.shell_cmd, - stdin=open(self.verify_script, "r")) + with open(self.verify_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + self.shell_cmd, + stdin=stdin_file) LOG.debug("action script of the operation is: {0}" .format(self.verify_script)) LOG.debug("action parameter the of operation is: {0}" .format(self.shell_cmd)) else: - exit_status, stdout, stderr = self.connection.execute( - "/bin/bash -s ", - stdin=open(self.verify_script, "r")) + with open(self.verify_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/bash -s ", + stdin=stdin_file) LOG.debug("action script of the operation is: {0}" .format(self.verify_script)) - LOG.debug("exit_status ,stdout : {0} ,{1}".format(exit_status, stdout)) + LOG.debug("exit_status ,stdout : %s ,%s", exit_status, stdout) if exit_status == 0 and stdout: self.actualResult = stdout LOG.debug("verifying resultchecker: {0}".format(self.key)) @@ -103,6 +106,6 @@ class GeneralResultChecker(BaseResultChecker): LOG.error(stderr) LOG.debug( - "verifying resultchecker: {0},the result is : {1}" - .format(self.key, self.success)) + "verifying resultchecker: %s,the result is : %s", self.key, + self.success) return self.success diff --git a/yardstick/benchmark/scenarios/availability/scenario_general.py b/yardstick/benchmark/scenarios/availability/scenario_general.py index 0a128aa09..b064c6724 100644 --- a/yardstick/benchmark/scenarios/availability/scenario_general.py +++ b/yardstick/benchmark/scenarios/availability/scenario_general.py @@ -22,7 +22,7 @@ class ScenarioGeneral(base.Scenario): def __init__(self, scenario_cfg, context_cfg): LOG.debug( - "scenario_cfg:%s context_cfg:%s" % (scenario_cfg, context_cfg)) + "scenario_cfg:%s context_cfg:%s", scenario_cfg, context_cfg) self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py index 10f2c4f45..46a197c3b 100755 --- a/yardstick/benchmark/scenarios/availability/serviceha.py +++ b/yardstick/benchmark/scenarios/availability/serviceha.py @@ -21,8 +21,8 @@ class ServiceHA(base.Scenario): def __init__(self, scenario_cfg, context_cfg): LOG.debug( - "scenario_cfg:%s context_cfg:%s" % - (scenario_cfg, context_cfg)) + "scenario_cfg:%s context_cfg:%s", + scenario_cfg, context_cfg) self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg self.setup_done = False diff --git a/yardstick/benchmark/scenarios/compute/cachestat.py b/yardstick/benchmark/scenarios/compute/cachestat.py index da4aa754f..20786ff61 100644 --- a/yardstick/benchmark/scenarios/compute/cachestat.py +++ b/yardstick/benchmark/scenarios/compute/cachestat.py @@ -75,22 +75,23 @@ class CACHEstat(base.Scenario): host = self.context_cfg['host'] user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get('ip', None) key_filename = host.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # copy scripts to host - self.client.run("cat > ~/cache_stat.sh", - stdin=open(self.target_script, 'rb')) + self.client._put_file_shell(self.target_script, '~/cache_stat.sh') self.setup_done = True def _execute_command(self, cmd): """Execute a command on server.""" - LOG.info("Executing: %s" % cmd) + LOG.info("Executing: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status: raise RuntimeError("Failed executing command: ", diff --git a/yardstick/benchmark/scenarios/compute/computecapacity.bash b/yardstick/benchmark/scenarios/compute/computecapacity.bash index 98d4b8fb5..68741a94f 100644 --- a/yardstick/benchmark/scenarios/compute/computecapacity.bash +++ b/yardstick/benchmark/scenarios/compute/computecapacity.bash @@ -9,13 +9,15 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -# Measure compute capacity and scale of a host +# compute capacity and scale of a host set -e # run capacity test run_capacity() { + #parameter used for HT(Hyper-Thread) check + HT_Para=2 # Number of CPUs CPU=$(grep 'physical id' /proc/cpuinfo | sort -u | wc -l) # Number of physical cores in a single CPU @@ -31,6 +33,12 @@ run_capacity() CACHE=$(grep 'cache size' /proc/cpuinfo | sort -u) CA=$(echo $CACHE | awk '/ /{printf "%s", $4}') CACHES=$[$CA * $CPU] + HT_Value=$[$HT_Para * $CORES] + if [ $HT_Value -eq $THREAD ]; then + HT_OPEN=1 + else + HT_OPEN=0 + fi } # write the result to stdout in json format @@ -41,7 +49,8 @@ output_json() \"Core_number\":\"$CORES\", \ \"Thread_number\":\"$THREAD\", \ \"Memory_size\": \"$ME\", \ - \"Cache_size\": \"$CACHES KB\" \ + \"Cache_size\": \"$CACHES KB\", \ + \"HT_Open\": \"$HT_OPEN\" \ }" } diff --git a/yardstick/benchmark/scenarios/compute/computecapacity.py b/yardstick/benchmark/scenarios/compute/computecapacity.py index 0d7d76143..7f0c58de1 100644 --- a/yardstick/benchmark/scenarios/compute/computecapacity.py +++ b/yardstick/benchmark/scenarios/compute/computecapacity.py @@ -40,15 +40,16 @@ class ComputeCapacity(base.Scenario): nodes = self.context_cfg['nodes'] node = nodes.get('host', None) host_user = node.get('user', 'ubuntu') + ssh_port = node.get('ssh_port', ssh.DEFAULT_PORT) host_ip = node.get('ip', None) host_pwd = node.get('password', 'root') LOG.debug("user:%s, host:%s", host_user, host_ip) - self.client = ssh.SSH(host_user, host_ip, password=host_pwd) + self.client = ssh.SSH(host_user, host_ip, password=host_pwd, + port=ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/computecapacity.sh", - stdin=open(self.target_script, 'rb')) + self.client._put_file_shell(self.target_script, '~/computecapacity.sh') self.setup_done = True diff --git a/yardstick/benchmark/scenarios/compute/cpuload.py b/yardstick/benchmark/scenarios/compute/cpuload.py index f45313e91..9d71038ef 100644 --- a/yardstick/benchmark/scenarios/compute/cpuload.py +++ b/yardstick/benchmark/scenarios/compute/cpuload.py @@ -67,10 +67,12 @@ class CPULoad(base.Scenario): host = self.context_cfg['host'] user = host.get('user', 'ubuntu') ip = host.get('ip', None) + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) key_filename = host.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # Check if mpstat prog is installed @@ -94,7 +96,7 @@ class CPULoad(base.Scenario): def _execute_command(self, cmd): """Execute a command on server.""" - LOG.info("Executing: %s" % cmd) + LOG.info("Executing: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status != 0: raise RuntimeError("Failed executing command: ", diff --git a/yardstick/benchmark/scenarios/compute/cyclictest.py b/yardstick/benchmark/scenarios/compute/cyclictest.py index 478b0a1a2..568e6e7df 100644 --- a/yardstick/benchmark/scenarios/compute/cyclictest.py +++ b/yardstick/benchmark/scenarios/compute/cyclictest.py @@ -69,14 +69,14 @@ class Cyclictest(base.Scenario): rpm_dir = setup_options["rpm_dir"] script_dir = setup_options["script_dir"] image_dir = setup_options["image_dir"] - LOG.debug("Send RPMs from %s to workspace %s" % - (rpm_dir, self.WORKSPACE)) + LOG.debug("Send RPMs from %s to workspace %s", + rpm_dir, self.WORKSPACE) client.put(rpm_dir, self.WORKSPACE, recursive=True) - LOG.debug("Send scripts from %s to workspace %s" % - (script_dir, self.WORKSPACE)) + LOG.debug("Send scripts from %s to workspace %s", + script_dir, self.WORKSPACE) client.put(script_dir, self.WORKSPACE, recursive=True) - LOG.debug("Send guest image from %s to workspace %s" % - (image_dir, self.WORKSPACE)) + LOG.debug("Send guest image from %s to workspace %s", + image_dir, self.WORKSPACE) client.put(image_dir, self.WORKSPACE, recursive=True) def _connect_host(self): @@ -93,14 +93,16 @@ class Cyclictest(base.Scenario): host = self.context_cfg["host"] user = host.get("user", "root") ip = host.get("ip", None) + ssh_port = host.get("ssh_port", 5555) key_filename = host.get("key_filename", "~/.ssh/id_rsa") LOG.debug("user:%s, host:%s", user, ip) - self.guest = ssh.SSH(user, ip, port=5555, key_filename=key_filename) + self.guest = ssh.SSH(user, ip, port=ssh_port, + key_filename=key_filename) self.guest.wait(timeout=600) def _run_setup_cmd(self, client, cmd): - LOG.debug("Run cmd: %s" % cmd) + LOG.debug("Run cmd: %s", cmd) status, stdout, stderr = client.execute(cmd) if status: if re.search(self.REBOOT_CMD_PATTERN, cmd): @@ -152,8 +154,8 @@ class Cyclictest(base.Scenario): self.target_script = pkg_resources.resource_filename( "yardstick.benchmark.scenarios.compute", Cyclictest.TARGET_SCRIPT) - self.guest.run("cat > ~/cyclictest_benchmark.sh", - stdin=open(self.target_script, "rb")) + self.guest._put_file_shell( + self.target_script, '~/cyclictest_benchmark.sh') self.setup_done = True diff --git a/yardstick/benchmark/scenarios/compute/lmbench.py b/yardstick/benchmark/scenarios/compute/lmbench.py index d3e802f3b..518840c09 100644 --- a/yardstick/benchmark/scenarios/compute/lmbench.py +++ b/yardstick/benchmark/scenarios/compute/lmbench.py @@ -77,20 +77,22 @@ class Lmbench(base.Scenario): Lmbench.LATENCY_CACHE_SCRIPT) host = self.context_cfg["host"] user = host.get("user", "ubuntu") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get("ip", None) key_filename = host.get('key_filename', "~/.ssh/id_rsa") LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # copy scripts to host - self.client.run("cat > ~/lmbench_latency.sh", - stdin=open(self.latency_target_script, 'rb')) - self.client.run("cat > ~/lmbench_bandwidth.sh", - stdin=open(self.bandwidth_target_script, 'rb')) - self.client.run("cat > ~/lmbench_latency_for_cache.sh", - stdin=open(self.latency_for_cache_script, 'rb')) + self.client._put_file_shell( + self.latency_target_script, '~/lmbench_latency.sh') + self.client._put_file_shell( + self.bandwidth_target_script, '~/lmbench_bandwidth.sh') + self.client._put_file_shell( + self.latency_for_cache_script, '~/lmbench_latency_for_cache.sh') self.setup_done = True def run(self, result): diff --git a/yardstick/benchmark/scenarios/compute/memload.py b/yardstick/benchmark/scenarios/compute/memload.py index bafd89617..e1ba93d02 100644 --- a/yardstick/benchmark/scenarios/compute/memload.py +++ b/yardstick/benchmark/scenarios/compute/memload.py @@ -48,18 +48,20 @@ class MEMLoad(base.Scenario): """Scenario setup.""" host = self.context_cfg['host'] user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get('ip', None) key_filename = host.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) self.setup_done = True def _execute_command(self, cmd): """Execute a command on server.""" - LOG.info("Executing: %s" % cmd) + LOG.info("Executing: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status: raise RuntimeError("Failed executing command: ", diff --git a/yardstick/benchmark/scenarios/compute/perf.py b/yardstick/benchmark/scenarios/compute/perf.py index f408e9cb4..8f1a4d630 100644 --- a/yardstick/benchmark/scenarios/compute/perf.py +++ b/yardstick/benchmark/scenarios/compute/perf.py @@ -47,16 +47,17 @@ class Perf(base.Scenario): 'yardstick.benchmark.scenarios.compute', Perf.TARGET_SCRIPT) host = self.context_cfg['host'] user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get('ip', None) key_filename = host.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/perf_benchmark.sh", - stdin=open(self.target_script, "rb")) + self.client._put_file_shell(self.target_script, '~/perf_benchmark.sh') self.setup_done = True diff --git a/yardstick/benchmark/scenarios/compute/plugintest.py b/yardstick/benchmark/scenarios/compute/plugintest.py index e41fb8399..e7ec91c5c 100644 --- a/yardstick/benchmark/scenarios/compute/plugintest.py +++ b/yardstick/benchmark/scenarios/compute/plugintest.py @@ -30,10 +30,12 @@ class PluginTest(base.Scenario): nodes = self.context_cfg['nodes'] node = nodes.get('host1', None) host_user = node.get('user', 'ubuntu') + host_ssh_port = node.get('ssh_port', ssh.DEFAULT_PORT) host_ip = node.get('ip', None) host_pwd = node.get('password', 'root') LOG.debug("user:%s, host:%s", host_user, host_ip) - self.client = ssh.SSH(host_user, host_ip, password=host_pwd) + self.client = ssh.SSH(host_user, host_ip, password=host_pwd, + port=host_ssh_port) self.client.wait(timeout=600) self.setup_done = True diff --git a/yardstick/benchmark/scenarios/compute/ramspeed.py b/yardstick/benchmark/scenarios/compute/ramspeed.py index 819ef769b..db70af90b 100644 --- a/yardstick/benchmark/scenarios/compute/ramspeed.py +++ b/yardstick/benchmark/scenarios/compute/ramspeed.py @@ -87,18 +87,20 @@ class Ramspeed(base.Scenario): host = self.context_cfg["host"] user = host.get("user", "ubuntu") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get("ip", None) key_filename = host.get('key_filename', "~/.ssh/id_rsa") LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # copy scripts to host - self.client.run("cat > ~/ramspeed_mark_benchmark.sh", - stdin=open(self.mark_target_script, 'rb')) - self.client.run("cat > ~/ramspeed_mem_benchmark.sh", - stdin=open(self.mem_target_script, 'rb')) + self.client._put_file_shell( + self.mark_target_script, '~/ramspeed_mark_benchmark.sh') + self.client._put_file_shell( + self.mem_target_script, '~/ramspeed_mem_benchmark.sh') self.setup_done = True def run(self, result): diff --git a/yardstick/benchmark/scenarios/compute/unixbench.py b/yardstick/benchmark/scenarios/compute/unixbench.py index e6318b92e..b22be29c9 100644 --- a/yardstick/benchmark/scenarios/compute/unixbench.py +++ b/yardstick/benchmark/scenarios/compute/unixbench.py @@ -67,16 +67,18 @@ class Unixbench(base.Scenario): host = self.context_cfg["host"] user = host.get("user", "ubuntu") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get("ip", None) key_filename = host.get('key_filename', "~/.ssh/id_rsa") LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # copy scripts to host - self.client.run("cat > ~/unixbench_benchmark.sh", - stdin=open(self.target_script, 'rb')) + self.client._put_file_shell( + self.target_script, '~/unixbench_benchmark.sh') self.setup_done = True @@ -152,5 +154,6 @@ def _test(): # pragma: no cover p.run(result) print result + if __name__ == '__main__': _test() diff --git a/yardstick/benchmark/scenarios/networking/iperf3.py b/yardstick/benchmark/scenarios/networking/iperf3.py index bb41c3df1..13fa0155b 100644 --- a/yardstick/benchmark/scenarios/networking/iperf3.py +++ b/yardstick/benchmark/scenarios/networking/iperf3.py @@ -56,21 +56,24 @@ For more info see http://software.es.net/iperf def setup(self): host = self.context_cfg['host'] host_user = host.get('user', 'ubuntu') + host_ssh_port = host.get('ssh_port', ssh.DEFAULT_PORT) host_ip = host.get('ip', None) host_key_filename = host.get('key_filename', '~/.ssh/id_rsa') target = self.context_cfg['target'] target_user = target.get('user', 'ubuntu') + target_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) target_ip = target.get('ip', None) target_key_filename = target.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, target:%s", target_user, target_ip) self.target = ssh.SSH(target_user, target_ip, - key_filename=target_key_filename) + key_filename=target_key_filename, + port=target_ssh_port) self.target.wait(timeout=600) LOG.info("user:%s, host:%s", host_user, host_ip) self.host = ssh.SSH(host_user, host_ip, - key_filename=host_key_filename) + key_filename=host_key_filename, port=host_ssh_port) self.host.wait(timeout=600) cmd = "iperf3 -s -D" diff --git a/yardstick/benchmark/scenarios/networking/netperf.py b/yardstick/benchmark/scenarios/networking/netperf.py index dcd4ef7b6..28f5bea56 100755 --- a/yardstick/benchmark/scenarios/networking/netperf.py +++ b/yardstick/benchmark/scenarios/networking/netperf.py @@ -62,27 +62,30 @@ class Netperf(base.Scenario): Netperf.TARGET_SCRIPT) host = self.context_cfg['host'] host_user = host.get('user', 'ubuntu') + host_ssh_port = host.get('ssh_port', ssh.DEFAULT_PORT) host_ip = host.get('ip', None) host_key_filename = host.get('key_filename', '~/.ssh/id_rsa') target = self.context_cfg['target'] target_user = target.get('user', 'ubuntu') + target_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) target_ip = target.get('ip', None) target_key_filename = target.get('key_filename', '~/.ssh/id_rsa') # netserver start automatically during the vm boot LOG.info("user:%s, target:%s", target_user, target_ip) self.server = ssh.SSH(target_user, target_ip, - key_filename=target_key_filename) + key_filename=target_key_filename, + port=target_ssh_port) self.server.wait(timeout=600) LOG.info("user:%s, host:%s", host_user, host_ip) self.client = ssh.SSH(host_user, host_ip, - key_filename=host_key_filename) + key_filename=host_key_filename, + port=host_ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/netperf.sh", - stdin=open(self.target_script, "rb")) + self.client._put_file_shell(self.target_script, '~/netperf.sh') self.setup_done = True @@ -174,5 +177,6 @@ def _test(): netperf.run(result) print result + if __name__ == '__main__': _test() diff --git a/yardstick/benchmark/scenarios/networking/netperf_benchmark.bash b/yardstick/benchmark/scenarios/networking/netperf_benchmark.bash index a425c5df0..f6245c9cd 100755 --- a/yardstick/benchmark/scenarios/networking/netperf_benchmark.bash +++ b/yardstick/benchmark/scenarios/networking/netperf_benchmark.bash @@ -12,6 +12,7 @@ set -e # Commandline arguments +OPTIONS_SIZE="$#" OPTIONS="$@" OUTPUT_FILE=/tmp/netperf-out.log @@ -24,14 +25,40 @@ run_netperf() # write the result to stdout in json format output_json() { - mean=$(awk '/\/s/{print $3}' $OUTPUT_FILE) - troughput=$(awk '/\/s/{print $1}' $OUTPUT_FILE) - unit=$(awk '/\/s/{print $2}' $OUTPUT_FILE) - echo -e "{ \ - \"mean_latency\":\"$mean\", \ - \"troughput\":\"$troughput\", \ - \"troughput_unit\":\"$unit\" \ - }" + #ARR=($OPTIONS) + #declare -p ARR + read -r -a ARR <<< "$OPTIONS" + opt_size=0 + while [ $opt_size -lt "$OPTIONS_SIZE" ] + do + if [ "${ARR[$opt_size]}" = "-O" ] + then + break + fi + opt_size=$((opt_size+1)) + done + opt_size=$((opt_size+1)) + out_opt="${ARR[$opt_size]}" + IFS=, read -r -a PARTS <<< "$out_opt" + #declare -p PARTS + part_num=${#PARTS[*]} + tran_num=0 + for f in "${PARTS[@]}" + do + array_name[$tran_num]=$(echo "$f" | tr '[A-Z]' '[a-z]') + tran_num=$((tran_num+1)) + done + read -r -a DATA_PARTS <<< "$(sed -n '$p' $OUTPUT_FILE)" + out_str="{" + for((i=0;i<part_num-1;i++)) + do + modify_str=\"${array_name[i]}\":\"${DATA_PARTS[i]}\", + out_str=$out_str$modify_str + done + modify_str=\"${array_name[part_num-1]}\":\"${DATA_PARTS[part_num-1]}\" + out_str=$out_str$modify_str"}" + + echo -e "$out_str" } # main entry @@ -44,4 +71,4 @@ main() output_json } -main
\ No newline at end of file +main diff --git a/yardstick/benchmark/scenarios/networking/netperf_install.bash b/yardstick/benchmark/scenarios/networking/netperf_install.bash index eaa9f530a..0e3808f9c 100755 --- a/yardstick/benchmark/scenarios/networking/netperf_install.bash +++ b/yardstick/benchmark/scenarios/networking/netperf_install.bash @@ -11,6 +11,13 @@ set -e +svc="netserver" +if pgrep $svc >/dev/null +then + echo "$svc have existed, exit!" + exit 0 +fi + echo "===Install netperf before test begin!!!===" cp /etc/apt/sources.list /etc/apt/sources.list_bkp cp /etc/resolv.conf /etc/resolv.conf_bkp @@ -30,3 +37,4 @@ sudo apt-get install -y netperf service netperf start echo "===Install netperf before test end!!!===" + diff --git a/yardstick/benchmark/scenarios/networking/netperf_node.py b/yardstick/benchmark/scenarios/networking/netperf_node.py index 87aa8d78d..a76982b6f 100755 --- a/yardstick/benchmark/scenarios/networking/netperf_node.py +++ b/yardstick/benchmark/scenarios/networking/netperf_node.py @@ -63,9 +63,11 @@ class NetperfNode(base.Scenario): NetperfNode.TARGET_SCRIPT) host = self.context_cfg['host'] host_user = host.get('user', 'ubuntu') + host_ssh_port = host.get('ssh_port', ssh.DEFAULT_PORT) host_ip = host.get('ip', None) target = self.context_cfg['target'] target_user = target.get('user', 'ubuntu') + target_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) target_ip = target.get('ip', None) self.target_ip = target.get('ip', None) host_password = host.get('password', None) @@ -75,18 +77,17 @@ class NetperfNode(base.Scenario): # netserver start automatically during the vm boot LOG.info("user:%s, target:%s", target_user, target_ip) self.server = ssh.SSH(target_user, target_ip, - password=target_password) + password=target_password, port=target_ssh_port) self.server.wait(timeout=600) LOG.info("user:%s, host:%s", host_user, host_ip) self.client = ssh.SSH(host_user, host_ip, - password=host_password) + password=host_password, port=host_ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/netperf.sh", - stdin=open(self.target_script, "rb")) - + with open(self.target_script, "rb") as file_run: + self.client.run("cat > ~/netperf.sh", stdin=file_run) # copy script to host and client self.install_script = pkg_resources.resource_filename( 'yardstick.benchmark.scenarios.networking', @@ -95,14 +96,14 @@ class NetperfNode(base.Scenario): 'yardstick.benchmark.scenarios.networking', NetperfNode.REMOVE_SCRIPT) - self.server.run("cat > ~/netperf_install.sh", - stdin=open(self.install_script, "rb")) - self.client.run("cat > ~/netperf_install.sh", - stdin=open(self.install_script, "rb")) - self.server.run("cat > ~/netperf_remove.sh", - stdin=open(self.remove_script, "rb")) - self.client.run("cat > ~/netperf_remove.sh", - stdin=open(self.remove_script, "rb")) + with open(self.install_script, "rb") as file_install: + self.server.run("cat > ~/netperf_install.sh", stdin=file_install) + with open(self.install_script, "rb") as file_install: + self.client.run("cat > ~/netperf_install.sh", stdin=file_install) + with open(self.remove_script, "rb") as file_remove: + self.server.run("cat > ~/netperf_remove.sh", stdin=file_remove) + with open(self.remove_script, "rb") as file_remove: + self.client.run("cat > ~/netperf_remove.sh", stdin=file_remove) self.server.execute("sudo bash netperf_install.sh") self.client.execute("sudo bash netperf_install.sh") @@ -129,10 +130,12 @@ class NetperfNode(base.Scenario): else: testlen = 20 - cmd_args = "-H %s -l %s -t %s" % (ipaddr, testlen, testname) + cmd_args = "-H %s -l %s -t %s -c -C" % (ipaddr, testlen, testname) # get test specific options - default_args = "-O 'THROUGHPUT,THROUGHPUT_UNITS,MEAN_LATENCY'" + output_opt = options.get( + "output_opt", "THROUGHPUT,THROUGHPUT_UNITS,MEAN_LATENCY") + default_args = "-O %s" % output_opt cmd_args += " -- %s" % default_args option_pair_list = [("send_msg_size", "-m"), ("recv_msg_size", "-M"), diff --git a/yardstick/benchmark/scenarios/networking/netutilization.py b/yardstick/benchmark/scenarios/networking/netutilization.py index ea43e6077..1ea92cca3 100644 --- a/yardstick/benchmark/scenarios/networking/netutilization.py +++ b/yardstick/benchmark/scenarios/networking/netutilization.py @@ -70,18 +70,20 @@ class NetUtilization(base.Scenario): """Scenario setup.""" host = self.context_cfg['host'] user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get('ip', None) key_filename = host.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) self.setup_done = True def _execute_command(self, cmd): """Execute a command on target.""" - LOG.info("Executing: %s" % cmd) + LOG.info("Executing: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status: raise RuntimeError("Failed executing command: ", diff --git a/yardstick/benchmark/scenarios/networking/networkcapacity.bash b/yardstick/benchmark/scenarios/networking/networkcapacity.bash index a18f97e0b..e2d3eb745 100644 --- a/yardstick/benchmark/scenarios/networking/networkcapacity.bash +++ b/yardstick/benchmark/scenarios/networking/networkcapacity.bash @@ -9,7 +9,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -# Measure compute capacity and scale of a host +# Measure network capacity and scale of a host set -e OUTPUT_FILE=/tmp/netperf-out.log diff --git a/yardstick/benchmark/scenarios/networking/networkcapacity.py b/yardstick/benchmark/scenarios/networking/networkcapacity.py index 57d3b5072..250f7eaf0 100644 --- a/yardstick/benchmark/scenarios/networking/networkcapacity.py +++ b/yardstick/benchmark/scenarios/networking/networkcapacity.py @@ -1,69 +1,70 @@ -##############################################################################
-# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-import pkg_resources
-import logging
-import json
-
-import yardstick.ssh as ssh
-from yardstick.benchmark.scenarios import base
-
-LOG = logging.getLogger(__name__)
-
-
-class NetworkCapacity(base.Scenario):
- """Measure Network capacity and scale.
-
- This scenario reads network status including number of connections,
- number of frames sent/received.
- """
- __scenario_type__ = "NetworkCapacity"
- TARGET_SCRIPT = "networkcapacity.bash"
-
- def __init__(self, scenario_cfg, context_cfg):
- self.scenario_cfg = scenario_cfg
- self.context_cfg = context_cfg
- self.setup_done = False
-
- def setup(self):
- """scenario setup"""
- self.target_script = pkg_resources.resource_filename(
- "yardstick.benchmark.scenarios.networking",
- NetworkCapacity.TARGET_SCRIPT)
-
- host = self.context_cfg['host']
- if host is None:
- raise RuntimeError('No right node.please check the configuration')
- host_user = host.get('user', 'ubuntu')
- host_ip = host.get('ip', None)
- host_pwd = host.get('password', None)
-
- LOG.debug("user:%s, host:%s", host_user, host_ip)
- self.client = ssh.SSH(host_user, host_ip, password=host_pwd)
- self.client.wait(timeout=600)
-
- # copy script to host
- self.client.run("cat > ~/networkcapacity.sh",
- stdin=open(self.target_script, 'rb'))
-
- self.setup_done = True
-
- def run(self, result):
- """execute the benchmark"""
-
- if not self.setup_done:
- self.setup()
-
- cmd = "sudo bash networkcapacity.sh"
-
- LOG.debug("Executing command: %s", cmd)
- status, stdout, stderr = self.client.execute(cmd)
- if status:
- raise RuntimeError(stderr)
-
- result.update(json.loads(stdout))
+############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import pkg_resources +import logging +import json + +import yardstick.ssh as ssh +from yardstick.benchmark.scenarios import base + +LOG = logging.getLogger(__name__) + + +class NetworkCapacity(base.Scenario): + """Measure Network capacity and scale. + + This scenario reads network status including number of connections, + number of frames sent/received. + """ + __scenario_type__ = "NetworkCapacity" + TARGET_SCRIPT = "networkcapacity.bash" + + def __init__(self, scenario_cfg, context_cfg): + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.setup_done = False + + def setup(self): + """scenario setup""" + self.target_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.networking", + NetworkCapacity.TARGET_SCRIPT) + + host = self.context_cfg['host'] + if host is None: + raise RuntimeError('No right node.please check the configuration') + host_user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) + host_ip = host.get('ip', None) + host_pwd = host.get('password', None) + + LOG.debug("user:%s, host:%s", host_user, host_ip) + self.client = ssh.SSH(host_user, host_ip, password=host_pwd, + port=ssh_port) + self.client.wait(timeout=600) + + # copy script to host + self.client._put_file_shell(self.target_script, '~/networkcapacity.sh') + + self.setup_done = True + + def run(self, result): + """execute the benchmark""" + + if not self.setup_done: + self.setup() + + cmd = "sudo bash networkcapacity.sh" + + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + result.update(json.loads(stdout)) diff --git a/yardstick/benchmark/scenarios/networking/ping.py b/yardstick/benchmark/scenarios/networking/ping.py index e09ea4a20..6e49a1437 100644 --- a/yardstick/benchmark/scenarios/networking/ping.py +++ b/yardstick/benchmark/scenarios/networking/ping.py @@ -39,6 +39,7 @@ class Ping(base.Scenario): 'yardstick.benchmark.scenarios.networking', Ping.TARGET_SCRIPT) host = self.context_cfg['host'] user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get('ip', None) key_filename = host.get('key_filename', '/root/.ssh/id_rsa') password = host.get('password', None) @@ -46,11 +47,13 @@ class Ping(base.Scenario): if password is not None: LOG.info("Log in via pw, user:%s, host:%s, pw:%s", user, ip, password) - self.connection = ssh.SSH(user, ip, password=password) + self.connection = ssh.SSH(user, ip, password=password, + port=ssh_port) else: LOG.info("Log in via key, user:%s, host:%s, key_filename:%s", user, ip, key_filename) - self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.connection.wait(timeout=600) @@ -76,9 +79,10 @@ class Ping(base.Scenario): target_vm = self.scenario_cfg['target'] LOG.debug("ping '%s' '%s'", options, dest) - exit_status, stdout, stderr = self.connection.execute( - "/bin/sh -s {0} {1}".format(dest, options), - stdin=open(self.target_script, "r")) + with open(self.target_script, "r") as stdin_file: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0} {1}".format(dest, options), + stdin=stdin_file) if exit_status != 0: raise RuntimeError(stderr) diff --git a/yardstick/benchmark/scenarios/networking/ping6.py b/yardstick/benchmark/scenarios/networking/ping6.py index e68756462..f4d23ce7b 100644 --- a/yardstick/benchmark/scenarios/networking/ping6.py +++ b/yardstick/benchmark/scenarios/networking/ping6.py @@ -37,28 +37,50 @@ class Ping6(base.Scenario): # pragma: no cover def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg + self.nodes = context_cfg['nodes'] + self.options = scenario_cfg['options'] self.setup_done = False self.run_done = False + self.external_network = self.options.get("external_network", "ext-net") + self.ping_options = "-s %s -c %s" % \ + (self.options.get("packetsize", '56'), + self.options.get("ping_count", '5')) + self.openrc = self.options.get("openrc", "/opt/admin-openrc.sh") + + def _ssh_host(self, node_name): + # ssh host + node = self.nodes.get(node_name, None) + user = node.get('user', 'ubuntu') + ssh_port = node.get("ssh_port", ssh.DEFAULT_PORT) + ip = node.get('ip', None) + pwd = node.get('password', None) + key_fname = node.get('key_filename', '/root/.ssh/id_rsa') + if pwd is not None: + LOG.debug("Log in via pw, user:%s, host:%s, password:%s", + user, ip, pwd) + self.client = ssh.SSH(user, ip, password=pwd, port=ssh_port) + else: + LOG.debug("Log in via key, user:%s, host:%s, key_filename:%s", + user, ip, key_fname) + self.client = ssh.SSH(user, ip, key_filename=key_fname, + port=ssh_port) + self.client.wait(timeout=60) def _pre_setup(self): for node_name in self.host_list: self._ssh_host(node_name) - self.client.run("cat > ~/pre_setup.sh", - stdin=open(self.pre_setup_script, "rb")) + self.client._put_file_shell( + self.pre_setup_script, '~/pre_setup.sh') status, stdout, stderr = self.client.execute( "sudo bash pre_setup.sh") - def _ssh_host(self, node_name): - # ssh host - print node_name - nodes = self.context_cfg['nodes'] - node = nodes.get(node_name, None) - host_user = node.get('user', 'ubuntu') - host_ip = node.get('ip', None) - host_pwd = node.get('password', 'root') - LOG.debug("user:%s, host:%s", host_user, host_ip) - self.client = ssh.SSH(host_user, host_ip, password=host_pwd) - self.client.wait(timeout=600) + def _get_controller_node(self, host_list): + for host_name in host_list: + node = self.nodes.get(host_name, None) + node_role = node.get('role', None) + if node_role == 'Controller': + return host_name + return None def setup(self): '''scenario setup''' @@ -82,33 +104,36 @@ class Ping6(base.Scenario): # pragma: no cover 'yardstick.benchmark.scenarios.networking', Ping6.RADVD_SCRIPT) - options = self.scenario_cfg['options'] - host_str = options.get("host", 'host1') + host_str = self.options.get("host", 'host1') self.host_list = host_str.split(',') self.host_list.sort() - pre_setup = options.get("pre_setup", True) + pre_setup = self.options.get("pre_setup", True) if pre_setup: self._pre_setup() - # ssh host1 - self._ssh_host(self.host_list[0]) - - self.client.run("cat > ~/metadata.txt", - stdin=open(self.ping6_metadata_script, "rb")) + # log in a contronller node to setup + controller_node_name = self._get_controller_node(self.host_list) + LOG.debug("The Controller Node is: %s", controller_node_name) + if controller_node_name is None: + LOG.exception("Can't find controller node in the context!!!") + self._ssh_host(controller_node_name) + self.client._put_file_shell( + self.ping6_metadata_script, '~/metadata.txt') # run script to setup ipv6 with nosdn or odl - sdn = options.get("sdn", 'nosdn') + sdn = self.options.get("sdn", 'nosdn') if 'odl' in sdn: - self.client.run("cat > ~/br-ex.radvd.conf", - stdin=open(self.ping6_radvd_script, "rb")) - self.client.run("cat > ~/setup_odl.sh", - stdin=open(self.setup_odl_script, "rb")) - cmd = "sudo bash setup_odl.sh" + self.client._put_file_shell( + self.ping6_radvd_script, '~/br-ex.radvd.conf') + self.client._put_file_shell( + self.setup_odl_script, '~/setup_odl.sh') + setup_bash_file = "setup_odl.sh" else: - self.client.run("cat > ~/setup.sh", - stdin=open(self.setup_script, "rb")) - cmd = "sudo bash setup.sh" - + self.client._put_file_shell(self.setup_script, '~/setup.sh') + setup_bash_file = "setup.sh" + cmd = "sudo bash %s %s %s" % \ + (setup_bash_file, self.openrc, self.external_network) + LOG.debug("Executing setup command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) self.setup_done = True @@ -123,19 +148,17 @@ class Ping6(base.Scenario): # pragma: no cover self.ping6_find_host_script = pkg_resources.resource_filename( 'yardstick.benchmark.scenarios.networking', Ping6.FIND_HOST_SCRIPT) - if not self.setup_done: - options = self.scenario_cfg['options'] - host_str = options.get("host", 'host1') + host_str = self.options.get("host", 'host1') self.host_list = host_str.split(',') self.host_list.sort() self._ssh_host(self.host_list[0]) # find ipv4-int-network1 to ssh VM - self.client.run("cat > ~/find_host.sh", - stdin=open(self.ping6_find_host_script, "rb")) - cmd = "sudo bash find_host.sh" - LOG.debug("Executing command: %s", cmd) + self.client._put_file_shell( + self.ping6_find_host_script, '~/find_host.sh') + cmd = "sudo bash find_host.sh %s" % self.openrc + LOG.debug("Executing find_host command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) host_name = stdout.strip() @@ -147,10 +170,9 @@ class Ping6(base.Scenario): # pragma: no cover stdin=open("/tmp/vRouterKey", "rb")) # run ping6 benchmark - self.client.run("cat > ~/ping6.sh", - stdin=open(self.ping6_script, "rb")) - cmd = "sudo bash ping6.sh" - LOG.debug("Executing command: %s", cmd) + self.client._put_file_shell(self.ping6_script, '~/ping6.sh') + cmd = "sudo bash ping6.sh %s %s" % (self.openrc, self.ping_options) + LOG.debug("Executing ping6 command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status: @@ -164,7 +186,7 @@ class Ping6(base.Scenario): # pragma: no cover assert result["rtt"] <= sla_max_rtt, \ "rtt %f > sla:max_rtt(%f); " % (result["rtt"], sla_max_rtt) else: - LOG.error("ping6 timeout") + LOG.error("ping6 timeout!!!") self.run_done = True def teardown(self): @@ -174,8 +196,7 @@ class Ping6(base.Scenario): # pragma: no cover 'yardstick.benchmark.scenarios.networking', Ping6.POST_TEARDOWN_SCRIPT) - options = self.scenario_cfg['options'] - host_str = options.get("host", 'node1') + host_str = self.options.get("host", 'node1') self.host_list = host_str.split(',') self.host_list.sort() @@ -185,12 +206,12 @@ class Ping6(base.Scenario): # pragma: no cover self.teardown_script = pkg_resources.resource_filename( 'yardstick.benchmark.scenarios.networking', Ping6.TEARDOWN_SCRIPT) - self.client.run("cat > ~/teardown.sh", - stdin=open(self.teardown_script, "rb")) - cmd = "sudo bash teardown.sh" + self.client._put_file_shell(self.teardown_script, '~/teardown.sh') + cmd = "sudo bash teardown.sh %s %s" % \ + (self.openrc, self.external_network) status, stdout, stderr = self.client.execute(cmd) - post_teardown = options.get("post_teardown", True) + post_teardown = self.options.get("post_teardown", True) if post_teardown: self._post_teardown() @@ -205,7 +226,7 @@ class Ping6(base.Scenario): # pragma: no cover def _post_teardown(self): for node_name in self.host_list: self._ssh_host(node_name) - self.client.run("cat > ~/post_teardown.sh", - stdin=open(self.post_teardown_script, "rb")) + self.client._put_file_shell( + self.post_teardown_script, '~/post_teardown.sh') status, stdout, stderr = self.client.execute( "sudo bash post_teardown.sh") diff --git a/yardstick/benchmark/scenarios/networking/ping6_benchmark.bash b/yardstick/benchmark/scenarios/networking/ping6_benchmark.bash index 16cb0f07e..a50e01f65 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_benchmark.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_benchmark.bash @@ -11,12 +11,18 @@ # Run a single ping6 command towards a ipv6 router set -e -source /opt/admin-openrc.sh +openrc=$1 +source $openrc +shift +ping6_options=$* chmod 600 vRouterKey + # TODO find host +vm1_ip=$(nova list|grep VM1 | awk -F [=] '{print $2}' | awk '{print $1}') +# echo "vm1_ip=$vm1_ip" wait_vm_ok() { retry=0 - until timeout 100s sudo ip netns exec qdhcp-$(neutron net-list | grep -w ipv4-int-network1 | awk '{print $2}') ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vRouterKey fedora@20.0.0.4 "exit" >/dev/null 2>&1 + until timeout 100s sudo ip netns exec qdhcp-$(neutron net-list | grep -w ipv4-int-network1 | awk '{print $2}') ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vRouterKey fedora@$vm1_ip "exit" >/dev/null 2>&1 do sleep 10 let retry+=1 @@ -29,4 +35,4 @@ wait_vm_ok() { } wait_vm_ok sleep 360 -sudo ip netns exec qdhcp-$(neutron net-list | grep -w ipv4-int-network1 | awk '{print $2}') ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vRouterKey fedora@20.0.0.4 "ping6 -c 1 2001:db8:0:1::1 | grep ttl | awk -F [=\ ] '{printf \$10}'"
\ No newline at end of file +sudo ip netns exec qdhcp-$(neutron net-list | grep -w ipv4-int-network1 | awk '{print $2}') ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vRouterKey fedora@$vm1_ip "ping6 $ping6_options 2001:db8:0:1::1 | grep rtt | awk -F [\/\ ] '{printf \$8}'" diff --git a/yardstick/benchmark/scenarios/networking/ping6_find_host.bash b/yardstick/benchmark/scenarios/networking/ping6_find_host.bash index 85c4b3898..db8dbe881 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_find_host.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_find_host.bash @@ -8,7 +8,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -source /opt/admin-openrc.sh -host_num=$(neutron dhcp-agent-list-hosting-net ipv4-int-network1 | grep True | awk -F [=\ ] '{printf $4}') > /tmp/ipv6.log -echo $host_num
\ No newline at end of file +openrc=$* +source $openrc +host_num=$(neutron dhcp-agent-list-hosting-net ipv4-int-network1 | grep True | head -1 | awk -F [=\ ] '{printf $4}' | grep -o '[0-9]\+') > /tmp/ipv6.log +echo "host$host_num" diff --git a/yardstick/benchmark/scenarios/networking/ping6_metadata.txt b/yardstick/benchmark/scenarios/networking/ping6_metadata.txt index 5dc08d30f..ad97b24c1 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_metadata.txt +++ b/yardstick/benchmark/scenarios/networking/ping6_metadata.txt @@ -45,7 +45,11 @@ write_files: owner: root:root - content: | TYPE="Ethernet" - BOOTPROTO=static + BOOTPROTO="dhcp" + DEFROUTE="no" + PEERDNS="no" + PEERROUTES="no" + IPV4_FAILURE_FATAL="no" IPV6INIT=yes IPV6ADDR="2001:db8:0:2::1/64" NAME=eth1 @@ -79,4 +83,4 @@ write_files: IPV6FORWARDING=yes path: /etc/sysconfig/network permissions: '0644' - owner: root:root
\ No newline at end of file + owner: root:root diff --git a/yardstick/benchmark/scenarios/networking/ping6_post_teardown.bash b/yardstick/benchmark/scenarios/networking/ping6_post_teardown.bash index f40d47d64..a0976e3d5 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_post_teardown.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_post_teardown.bash @@ -8,10 +8,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -sed -i 's/enable_security_group= False/enable_security_group = True/g' /etc/neutron/plugins/ml2/ml2_conf.ini -sed -i 3d /etc/neutron/plugins/ml2/ml2_conf.ini -sed -i 's/firewall_driver= neutron.agent.firewall.NoopFirewallDriver/firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver/g' /etc/neutron/plugins/ml2/ml2_conf.ini -sed -i 's/security_group_api= nova/security_group_api = neutron/g' /etc/nova/nova.conf +cp /etc/neutron/plugins/ml2/ml2_conf.ini_bkp /etc/neutron/plugins/ml2/ml2_conf.ini service neutron-l3-agent restart service neutron-dhcp-agent restart @@ -23,4 +20,4 @@ service nova-conductor restart service nova-consoleauth restart service nova-novncproxy restart service nova-scheduler restart -service nova-compute restart
\ No newline at end of file +service nova-compute restart diff --git a/yardstick/benchmark/scenarios/networking/ping6_pre_setup.bash b/yardstick/benchmark/scenarios/networking/ping6_pre_setup.bash index 4a781d21e..7fa60592d 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_pre_setup.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_pre_setup.bash @@ -9,12 +9,34 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -sed -i 's/enable_security_group = True/enable_security_group= False/g' /etc/neutron/plugins/ml2/ml2_conf.ini -sed -i '2a extension_drivers = port_security' /etc/neutron/plugins/ml2/ml2_conf.ini -sed -i 's/firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver/firewall_driver= neutron.agent.firewall.NoopFirewallDriver/g' /etc/neutron/plugins/ml2/ml2_conf.ini -sed -i 's/security_group_api = neutron/security_group_api= nova/g' /etc/nova/nova.conf +ML2_CONF_FILE="/etc/neutron/plugins/ml2/ml2_conf.ini" +NOVA_CONF_FILE="/etc/nova/nova.conf" +cp $ML2_CONF_FILE ${ML2_CONF_FILE}_bkp + +agent_line_num=$(grep -n '\[agent\]' $ML2_CONF_FILE | awk -F [:] '{print $1}') +if [ -z "$agent_line_num" ] +then + echo "[agent]" >> $ML2_CONF_FILE + agent_line_num=$(wc -l $ML2_CONF_FILE | awk '{print $1}') +fi +sed -i "${agent_line_num}a prevent_arp_spoofing = False" $ML2_CONF_FILE + +sed -i 's/firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver/firewall_driver= neutron.agent.firewall.NoopFirewallDriver/g' $ML2_CONF_FILE + +#check parameters +echo "check if parameters ok" +echo $ML2_CONF_FILE +grep -n 'enable_security_group = True' $ML2_CONF_FILE +grep -n 'extension_drivers = port_security' $ML2_CONF_FILE +grep -n 'prevent_arp_spoofing = False' $ML2_CONF_FILE +echo $NOVA_CONF_FILE +grep -n 'security_group_api = neutron' $NOVA_CONF_FILE +grep -n 'firewall_driver = nova.virt.firewall.NoopFirewallDriver' $NOVA_CONF_FILE +echo "check parameters end" + +# restart nova and neutron service service neutron-l3-agent restart service neutron-dhcp-agent restart service neutron-metadata-agent restart diff --git a/yardstick/benchmark/scenarios/networking/ping6_setup.bash b/yardstick/benchmark/scenarios/networking/ping6_setup.bash index 658e1d3cf..592ced3df 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_setup.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_setup.bash @@ -11,8 +11,17 @@ # download and create image -source /opt/admin-openrc.sh -wget https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-22-20150521.x86_64.qcow2 >/dev/null 2>&1 +openrc=$1 +external_network=$2 +echo "openrc=$openrc" +echo "external_network=$external_network" +echo "nameserver 8.8.4.4" >> /etc/resolv.conf +source $openrc + +fedora_img="Fedora-Cloud-Base-22-20150521.x86_64.qcow2" +if [ ! -f "$fedora_img" ]; then + wget https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/${fedora_img} >/dev/null 2>&1 +fi glance image-create --name 'Fedora22' --disk-format qcow2 \ --container-format bare --file ./Fedora-Cloud-Base-22-20150521.x86_64.qcow2 @@ -24,8 +33,8 @@ neutron router-create ipv6-router # create (ipv4,ipv6)router and net and subnet -neutron net-create --port_security_enabled=False ipv4-int-network1 -neutron net-create --port_security_enabled=False ipv6-int-network2 +neutron net-create ipv4-int-network1 +neutron net-create ipv6-int-network2 # Create IPv4 subnet and associate it to ipv4-router neutron subnet-create --name ipv4-int-subnet1 \ @@ -33,8 +42,8 @@ neutron subnet-create --name ipv4-int-subnet1 \ neutron router-interface-add ipv4-router ipv4-int-subnet1 # Associate the net04_ext to the Neutron routers -neutron router-gateway-set ipv6-router ext-net -neutron router-gateway-set ipv4-router ext-net +neutron router-gateway-set ipv6-router $external_network +neutron router-gateway-set ipv4-router $external_network # Create two subnets, one IPv4 subnet ipv4-int-subnet2 and # one IPv6 subnet ipv6-int-subnet2 in ipv6-int-network2, and associate both subnets to ipv6-router @@ -78,4 +87,13 @@ nova boot --image Fedora22 --flavor m1.small \ --nic port-id=$(neutron port-list | grep -w eth0-VM2 | awk '{print $2}') \ --key-name vRouterKey VM2 +sleep 60 + nova list +# disable eth0-VM1, eth0-VM2, eth0-vRouter, eth1-vRouter port-security +for port in eth0-VM1 eth0-VM2 eth0-vRouter eth1-vRouter +do + neutron port-update --no-security-groups $port + neutron port-update $port --port-security-enabled=False + neutron port-show $port | grep port_security_enabled +done diff --git a/yardstick/benchmark/scenarios/networking/ping6_setup_with_odl.bash b/yardstick/benchmark/scenarios/networking/ping6_setup_with_odl.bash index f9a2c4094..21566d2aa 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_setup_with_odl.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_setup_with_odl.bash @@ -11,7 +11,11 @@ # need to debug # download and create image -source /opt/admin-openrc.sh +openrc=$1 +external_network=$2 +echo "openrc=$openrc" +echo "external_network=$external_network" +source $openrc wget https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-22-20150521.x86_64.qcow2 glance image-create --name 'Fedora22' --disk-format qcow2 \ --container-format bare --file ./Fedora-Cloud-Base-22-20150521.x86_64.qcow2 @@ -21,8 +25,8 @@ neutron router-create ipv4-router neutron router-create ipv6-router # Associate the net04_ext to the Neutron routers -neutron router-gateway-set ipv6-router ext-net -neutron router-gateway-set ipv4-router ext-net +neutron router-gateway-set ipv6-router $external_network +neutron router-gateway-set ipv4-router $external_network # create two ipv4 networks with associated subnets neutron net-create ipv4-int-network1 diff --git a/yardstick/benchmark/scenarios/networking/ping6_teardown.bash b/yardstick/benchmark/scenarios/networking/ping6_teardown.bash index 33eff5ca7..2fe3ef2fb 100644 --- a/yardstick/benchmark/scenarios/networking/ping6_teardown.bash +++ b/yardstick/benchmark/scenarios/networking/ping6_teardown.bash @@ -8,7 +8,11 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -source /opt/admin-openrc.sh +openrc=$1 +echo "openrc=$openrc" +source $openrc +external_network=$2 +echo "external_network=$external_network" # delete VM nova delete VM1 nova delete VM2 @@ -36,8 +40,8 @@ neutron subnet-delete --name ipv6-int-subnet2 neutron subnet-delete --name ipv4-int-subnet2 #clear gateway -neutron router-gateway-clear ipv4-router ext-net -neutron router-gateway-clear ipv6-router ext-net +neutron router-gateway-clear ipv4-router $external_network +neutron router-gateway-clear ipv6-router $external_network #delete ipv4 router interface neutron router-interface-delete ipv4-router ipv4-int-subnet1 diff --git a/yardstick/benchmark/scenarios/networking/pktgen.py b/yardstick/benchmark/scenarios/networking/pktgen.py index 9dac4c90c..e2df706a2 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen.py +++ b/yardstick/benchmark/scenarios/networking/pktgen.py @@ -49,26 +49,29 @@ class Pktgen(base.Scenario): Pktgen.TARGET_SCRIPT) host = self.context_cfg['host'] host_user = host.get('user', 'ubuntu') + host_ssh_port = host.get('ssh_port', ssh.DEFAULT_PORT) host_ip = host.get('ip', None) host_key_filename = host.get('key_filename', '~/.ssh/id_rsa') target = self.context_cfg['target'] target_user = target.get('user', 'ubuntu') + target_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) target_ip = target.get('ip', None) target_key_filename = target.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, target:%s", target_user, target_ip) self.server = ssh.SSH(target_user, target_ip, - key_filename=target_key_filename) + key_filename=target_key_filename, + port=target_ssh_port) self.server.wait(timeout=600) LOG.info("user:%s, host:%s", host_user, host_ip) self.client = ssh.SSH(host_user, host_ip, - key_filename=host_key_filename) + key_filename=host_key_filename, + port=host_ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/pktgen.sh", - stdin=open(self.target_script, "rb")) + self.client._put_file_shell(self.target_script, '~/pktgen.sh') self.setup_done = True @@ -169,5 +172,6 @@ def _test(): p.run(result) print result + if __name__ == '__main__': _test() diff --git a/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py b/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py index 86585eca3..503ea97e1 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py +++ b/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py @@ -45,29 +45,32 @@ class PktgenDPDKLatency(base.Scenario): PktgenDPDKLatency.TESTPMD_SCRIPT) host = self.context_cfg['host'] host_user = host.get('user', 'ubuntu') + host_ssh_port = host.get('ssh_port', ssh.DEFAULT_PORT) host_ip = host.get('ip', None) host_key_filename = host.get('key_filename', '~/.ssh/id_rsa') target = self.context_cfg['target'] target_user = target.get('user', 'ubuntu') + target_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) target_ip = target.get('ip', None) target_key_filename = target.get('key_filename', '~/.ssh/id_rsa') LOG.info("user:%s, target:%s", target_user, target_ip) self.server = ssh.SSH(target_user, target_ip, - key_filename=target_key_filename) + key_filename=target_key_filename, + port=target_ssh_port) self.server.wait(timeout=600) # copy script to host - self.server.run("cat > ~/testpmd_fwd.sh", - stdin=open(self.testpmd_script, "rb")) + self.server._put_file_shell(self.testpmd_script, '~/testpmd_fwd.sh') LOG.info("user:%s, host:%s", host_user, host_ip) self.client = ssh.SSH(host_user, host_ip, - key_filename=host_key_filename) + key_filename=host_key_filename, + port=host_ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/pktgen_dpdk.sh", - stdin=open(self.pktgen_dpdk_script, "rb")) + self.client._put_file_shell( + self.pktgen_dpdk_script, '~/pktgen_dpdk.sh') self.setup_done = True self.testpmd_args = '' @@ -149,7 +152,7 @@ class PktgenDPDKLatency(base.Scenario): latency_sum = 0 for i in latency_list: latency_sum += int(i) - avg_latency = latency_sum/len(latency_list) + avg_latency = latency_sum / len(latency_list) result.update({"avg_latency": avg_latency}) diff --git a/yardstick/benchmark/scenarios/networking/sfc.py b/yardstick/benchmark/scenarios/networking/sfc.py index a126bb52a..1bd99b957 100644 --- a/yardstick/benchmark/scenarios/networking/sfc.py +++ b/yardstick/benchmark/scenarios/networking/sfc.py @@ -41,15 +41,16 @@ class Sfc(base.Scenario): # pragma: no cover target = self.context_cfg['target'] target_user = target.get('user', 'root') + target_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) target_pwd = target.get('password', 'opnfv') target_ip = target.get('ip', None) ''' webserver start automatically during the vm boot ''' LOG.info("user:%s, target:%s", target_user, target_ip) - self.server = ssh.SSH(target_user, target_ip, password=target_pwd) + self.server = ssh.SSH(target_user, target_ip, password=target_pwd, + port=target_ssh_port) self.server.wait(timeout=600) - self.server.run("cat > ~/server.sh", - stdin=open(self.server_script, "rb")) + self.server._put_file_shell(self.server_script, '~/server.sh') cmd_server = "sudo bash server.sh" LOG.debug("Executing command: %s", cmd_server) status, stdout, stderr = self.server.execute(cmd_server) @@ -59,11 +60,13 @@ class Sfc(base.Scenario): # pragma: no cover target = self.context_cfg['target'] SF1_user = target.get('user', 'root') + SF1_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) SF1_pwd = target.get('password', 'opnfv') SF1_ip = ips[0] LOG.info("user:%s, host:%s", SF1_user, SF1_ip) - self.server = ssh.SSH(SF1_user, SF1_ip, password=SF1_pwd) + self.server = ssh.SSH(SF1_user, SF1_ip, password=SF1_pwd, + port=SF1_ssh_port) self.server.wait(timeout=600) cmd_SF1 = ("nohup python vxlan_tool.py -i eth0 " "-d forward -v off -b 80 &") @@ -74,11 +77,13 @@ class Sfc(base.Scenario): # pragma: no cover LOG.debug("HTTP firewall started") SF2_user = target.get('user', 'root') + SF2_ssh_port = target.get('ssh_port', ssh.DEFAULT_PORT) SF2_pwd = target.get('password', 'opnfv') SF2_ip = ips[1] LOG.info("user:%s, host:%s", SF2_user, SF2_ip) - self.server = ssh.SSH(SF2_user, SF2_ip, password=SF2_pwd) + self.server = ssh.SSH(SF2_user, SF2_ip, password=SF2_pwd, + port=SF2_ssh_port) self.server.wait(timeout=600) cmd_SF2 = ("nohup python vxlan_tool.py -i eth0 " "-d forward -v off -b 22 &") @@ -95,11 +100,13 @@ class Sfc(base.Scenario): # pragma: no cover ''' Creating client and server VMs to perform the test''' host = self.context_cfg['host'] host_user = host.get('user', 'root') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) host_pwd = host.get('password', 'opnfv') host_ip = host.get('ip', None) LOG.info("user:%s, host:%s", host_user, host_ip) - self.client = ssh.SSH(host_user, host_ip, password=host_pwd) + self.client = ssh.SSH(host_user, host_ip, password=host_pwd, + port=ssh_port) self.client.wait(timeout=600) if not self.setup_done: # pragma: no cover diff --git a/yardstick/benchmark/scenarios/networking/vsperf.py b/yardstick/benchmark/scenarios/networking/vsperf.py index d3123083a..4f4ef21eb 100644 --- a/yardstick/benchmark/scenarios/networking/vsperf.py +++ b/yardstick/benchmark/scenarios/networking/vsperf.py @@ -32,14 +32,11 @@ class Vsperf(base.Scenario): the valid values are "rfc2544", "continuous", "back2back" type: string default: "rfc2544" - pkt_sizes - a packet size for which test should be executed; - Multiple packet sizes can be tested by modification of Sequence runner + frame_size - a frame size for which test should be executed; + Multiple frame sizes can be tested by modification of sequence runner section inside TC YAML definition. type: string default: "64" - duration - sets duration for which traffic will be generated - type: int - default: 30 bidirectional - speficies if traffic will be uni (False) or bi-directional (True) type: string @@ -47,9 +44,6 @@ class Vsperf(base.Scenario): iload - specifies frame rate type: string default: 100 - rfc2544_trials - the number of trials performed for each packet size - type: string - default: NA multistream - the number of simulated streams type: string default: 0 (disabled) @@ -57,11 +51,24 @@ class Vsperf(base.Scenario): the valid values are "L4", "L3" and "L2" type: string default: "L4" - conf-file - path to the vsperf configuration file, which will be uploaded - to the VM + test_params - specifies a string with a list of vsperf configuration + parameters, which will be passed to the '--test-params' CLI argument; + Parameters should be stated in the form of 'param=value' and separated + by a semicolon. Please check VSPERF documentation for details about + available configuration parameters and their data types. + In case that both 'test_params' and 'conf_file' are specified, + then values from 'test_params' will override values defined + in the configuration file. + type: string + default: NA + conf_file - path to the vsperf configuration file, which will be uploaded + to the VM; + In case that both 'test_params' and 'conf_file' are specified, + then values from 'test_params' will override values defined + in configuration file. type: string default: NA - setup-script - path to the setup script, which will be executed during + setup_script - path to the setup script, which will be executed during setup and teardown phases type: string default: NA @@ -80,8 +87,6 @@ class Vsperf(base.Scenario): """ __scenario_type__ = "Vsperf" - VSPERF_CONF = '~/vsperf-yardstick.conf' - def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg @@ -93,17 +98,23 @@ class Vsperf(base.Scenario): None) self.br_ex = self.scenario_cfg['options'].get('external_bridge', 'br-ex') - self.vsperf_conf = os.path.expanduser( - self.scenario_cfg['options'].get('conf-file', Vsperf.VSPERF_CONF)) - self.setup_script = self.scenario_cfg['options'].get('setup-script', + self.vsperf_conf = self.scenario_cfg['options'].get('conf_file', None) + if self.vsperf_conf: + self.vsperf_conf = os.path.expanduser(self.vsperf_conf) + + self.setup_script = self.scenario_cfg['options'].get('setup_script', None) if self.setup_script: self.setup_script = os.path.expanduser(self.setup_script) + self.test_params = self.scenario_cfg['options'].get('test-params', + None) + def setup(self): '''scenario setup''' vsperf = self.context_cfg['host'] vsperf_user = vsperf.get('user', 'ubuntu') + vsperf_ssh_port = vsperf.get('ssh_port', ssh.DEFAULT_PORT) vsperf_password = vsperf.get('password', 'ubuntu') vsperf_ip = vsperf.get('ip', None) @@ -118,13 +129,12 @@ class Vsperf(base.Scenario): # copy vsperf conf to VM LOG.info("user:%s, host:%s", vsperf_user, vsperf_ip) self.client = ssh.SSH(vsperf_user, vsperf_ip, - password=vsperf_password) + password=vsperf_password, port=vsperf_ssh_port) # traffic generation could last long self.client.wait(timeout=1800) # copy script to host - self.client.run("cat > ~/vsperf.conf", - stdin=open(self.vsperf_conf, "rb")) + self.client._put_file_shell(self.vsperf_conf, '~/vsperf.conf') # execute external setup script if self.setup_script: @@ -165,18 +175,26 @@ class Vsperf(base.Scenario): options = self.scenario_cfg['options'] test_params = [] test_params.append(add_test_params(options, "traffic_type", "rfc2544")) - test_params.append(add_test_params(options, "pkt_sizes", "64")) - test_params.append(add_test_params(options, "duration", None)) test_params.append(add_test_params(options, "bidirectional", "False")) test_params.append(add_test_params(options, "iload", 100)) - test_params.append(add_test_params(options, "rfc2544_trials", None)) test_params.append(add_test_params(options, "multistream", None)) test_params.append(add_test_params(options, "stream_type", None)) + if 'frame_size' in options: + test_params.append("%s=(%s,)" % ('TRAFFICGEN_PKT_SIZES', + options['frame_size'])) + if 'test_params' in options: + test_params.append(options['test_params']) + + # filter empty parameters and escape quotes and double quotes + test_params = [tp.replace('"', '\\"').replace("'", "\\'") + for tp in test_params if tp] # execute vsperf cmd = "source ~/vsperfenv/bin/activate ; cd vswitchperf ; " - cmd += "./vsperf --mode trafficgen --conf-file ~/vsperf.conf " - cmd += "--test-params=\"%s\"" % (';'.join(filter(None, test_params))) + cmd += "./vsperf --mode trafficgen " + if self.vsperf_conf: + cmd += "--conf-file ~/vsperf.conf " + cmd += "--test-params=\"%s\"" % (';'.join(test_params)) LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) diff --git a/yardstick/benchmark/scenarios/storage/fio.py b/yardstick/benchmark/scenarios/storage/fio.py index a8d27faba..4e004235d 100644 --- a/yardstick/benchmark/scenarios/storage/fio.py +++ b/yardstick/benchmark/scenarios/storage/fio.py @@ -60,16 +60,17 @@ class Fio(base.Scenario): Fio.TARGET_SCRIPT) host = self.context_cfg["host"] user = host.get("user", "root") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) ip = host.get("ip", None) key_filename = host.get("key_filename", "~/.ssh/id_rsa") LOG.info("user:%s, host:%s", user, ip) - self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/fio.sh", - stdin=open(self.target_script, "rb")) + self.client._put_file_shell(self.target_script, '~/fio.sh') self.setup_done = True diff --git a/yardstick/benchmark/scenarios/storage/storagecapacity.py b/yardstick/benchmark/scenarios/storage/storagecapacity.py index 49e3a0339..bf5bc2810 100644 --- a/yardstick/benchmark/scenarios/storage/storagecapacity.py +++ b/yardstick/benchmark/scenarios/storage/storagecapacity.py @@ -54,16 +54,17 @@ class StorageCapacity(base.Scenario): if host is None: raise RuntimeError('No right node.Please check the configuration') host_user = host.get('user', 'ubuntu') + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) host_ip = host.get('ip', None) host_pwd = host.get('password', 'root') LOG.debug("user:%s, host:%s", host_user, host_ip) - self.client = ssh.SSH(host_user, host_ip, password=host_pwd) + self.client = ssh.SSH(host_user, host_ip, password=host_pwd, + port=ssh_port) self.client.wait(timeout=600) # copy script to host - self.client.run("cat > ~/storagecapacity.sh", - stdin=open(self.target_script, 'rb')) + self.client._put_file_shell(self.target_script, '~/storagecapacity.sh') self.setup_done = True @@ -107,7 +108,7 @@ class StorageCapacity(base.Scenario): for i in range(len(device_name_arr)): r[device_name_arr[i]] = {"min_util": min_util_arr[i], "max_util": max_util_arr[i], - "avg_util": avg_util_arr[i]/count} + "avg_util": avg_util_arr[i] / count} return r def run(self, result): diff --git a/yardstick/benchmark/scenarios/storage/storperf.py b/yardstick/benchmark/scenarios/storage/storperf.py index d39c23aa2..72ceff7ce 100644 --- a/yardstick/benchmark/scenarios/storage/storperf.py +++ b/yardstick/benchmark/scenarios/storage/storperf.py @@ -54,6 +54,7 @@ class StorPerf(base.Scenario): def __init__(self, scenario_cfg, context_cfg): """Scenario construction.""" + super(StorPerf, self).__init__() self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg self.options = self.scenario_cfg["options"] @@ -75,8 +76,8 @@ class StorPerf(base.Scenario): setup_query_content = json.loads(setup_query.content) if setup_query_content["stack_created"]: self.setup_done = True - LOG.debug("stack_created: %s" - % setup_query_content["stack_created"]) + LOG.debug("stack_created: %s", + setup_query_content["stack_created"]) def setup(self): """Set the configuration.""" @@ -85,45 +86,44 @@ class StorPerf(base.Scenario): "agent_image", "volume_size"] for env_argument in env_args_payload_list: - if env_argument in self.options: + try: env_args[env_argument] = self.options[env_argument] + except KeyError: + pass - LOG.info("Creating a stack on node %s with parameters %s" % - (self.target, env_args)) + LOG.info("Creating a stack on node %s with parameters %s", + self.target, env_args) setup_res = requests.post('http://%s:5000/api/v1.0/configurations' % self.target, json=env_args) setup_res_content = json.loads(setup_res.content) - if setup_res.status_code == 400: + if setup_res.status_code != 200: raise RuntimeError("Failed to create a stack, error message:", setup_res_content["message"]) elif setup_res.status_code == 200: - LOG.info("stack_id: %s" % setup_res_content["stack_id"]) + LOG.info("stack_id: %s", setup_res_content["stack_id"]) while not self.setup_done: self._query_setup_state() time.sleep(self.query_interval) - # TODO: Support Storperf job status. + def _query_job_state(self, job_id): + """Query the status of the supplied job_id and report on metrics""" + LOG.info("Fetching report for %s...", job_id) + report_res = requests.get('http://{}:5000/api/v1.0/jobs'.format + (self.target), params={'id': job_id}) - # def _query_job_state(self, job_id): - # """Query the status of the supplied job_id and report on metrics""" - # LOG.info("Fetching report for %s..." % job_id) - # report_res = requests.get('http://%s:5000/api/v1.0/jobs?id=%s' % - # (self.target, job_id)) + report_res_content = json.loads(report_res.content) - # report_res_content = json.loads(report_res.content) + if report_res.status_code != 200: + raise RuntimeError("Failed to fetch report, error message:", + report_res_content["message"]) + else: + job_status = report_res_content["status"] - # if report_res.status_code == 400: - # raise RuntimeError("Failed to fetch report, error message:", - # report_res_content["message"]) - # else: - # job_status = report_res_content["status"] - - # LOG.debug("Job is: %s..." % job_status) - # if job_status == "completed": - # self.job_completed = True + LOG.debug("Job is: %s...", job_status) + self.job_completed = job_status == "completed" # TODO: Support using StorPerf ReST API to read Job ETA. @@ -145,37 +145,36 @@ class StorPerf(base.Scenario): "target", "nossd", "nowarm", "workload"] for job_argument in job_args_payload_list: - if job_argument in self.options: + try: job_args[job_argument] = self.options[job_argument] + except KeyError: + pass - LOG.info("Starting a job with parameters %s" % job_args) + LOG.info("Starting a job with parameters %s", job_args) job_res = requests.post('http://%s:5000/api/v1.0/jobs' % self.target, json=job_args) job_res_content = json.loads(job_res.content) - if job_res.status_code == 400: + if job_res.status_code != 200: raise RuntimeError("Failed to start a job, error message:", job_res_content["message"]) elif job_res.status_code == 200: job_id = job_res_content["job_id"] - LOG.info("Started job id: %s..." % job_id) + LOG.info("Started job id: %s...", job_id) + + while not self.job_completed: + self._query_job_state(job_id) + time.sleep(self.query_interval) - time.sleep(self.timeout) terminate_res = requests.delete('http://%s:5000/api/v1.0/jobs' % self.target) - if terminate_res.status_code == 400: + if terminate_res.status_code != 200: terminate_res_content = json.loads(terminate_res.content) raise RuntimeError("Failed to start a job, error message:", terminate_res_content["message"]) - # TODO: Support Storperf job status. - - # while not self.job_completed: - # self._query_job_state(job_id) - # time.sleep(self.query_interval) - # TODO: Support using ETA to polls for completion. # Read ETA, next poll in 1/2 ETA time slot. # If ETA is greater than the maximum allowed job time, diff --git a/yardstick/cmd/cli.py b/yardstick/cmd/cli.py index dd74836cb..beaa187aa 100644 --- a/yardstick/cmd/cli.py +++ b/yardstick/cmd/cli.py @@ -19,11 +19,13 @@ from pkg_resources import get_distribution from argparse import RawDescriptionHelpFormatter from oslo_config import cfg +from yardstick import _init_logging, LOG from yardstick.cmd.commands import task from yardstick.cmd.commands import runner from yardstick.cmd.commands import scenario from yardstick.cmd.commands import testcase from yardstick.cmd.commands import plugin +from yardstick.cmd.commands import env CONF = cfg.CONF cli_opts = [ @@ -62,10 +64,12 @@ class YardstickCLI(): 'runner': runner.RunnerCommands, 'scenario': scenario.ScenarioCommands, 'testcase': testcase.TestcaseCommands, - 'plugin': plugin.PluginCommands + 'plugin': plugin.PluginCommands, + 'env': env.EnvCommand } def __init__(self): + self.opts = [] self._version = 'yardstick version %s ' % \ get_distribution('yardstick').version @@ -101,8 +105,7 @@ class YardstickCLI(): cmd_subparsers = subparser.add_subparsers(title='subcommands') self._find_actions(cmd_subparsers, command_object) - def main(self, argv): - '''run the command line interface''' + def _register_cli_opt(self): # register subcommands to parse additional command line arguments def parser(subparsers): @@ -112,22 +115,67 @@ class YardstickCLI(): title="Command categories", help="Available categories", handler=parser) - CONF.register_cli_opt(category_opt) + self._register_opt(category_opt) + + def _register_opt(self, opt): + + CONF.register_cli_opt(opt) + self.opts.append(opt) + + def _load_cli_config(self, argv): # load CLI args and config files CONF(argv, project="yardstick", version=self._version, default_config_files=find_config_files(CONFIG_SEARCH_PATHS)) - # handle global opts - logger = logging.getLogger('yardstick') - logger.setLevel(logging.WARNING) + def _handle_global_opts(self): + _init_logging() if CONF.verbose: - logger.setLevel(logging.INFO) + LOG.setLevel(logging.INFO) if CONF.debug: - logger.setLevel(logging.DEBUG) + LOG.setLevel(logging.DEBUG) + + def _dispath_func_notask(self): # dispatch to category parser func = CONF.category.func func(CONF.category) + + def _dispath_func_task(self, task_id): + + # dispatch to category parser + func = CONF.category.func + func(CONF.category, task_id=task_id) + + def _clear_config_opts(self): + + CONF.clear() + CONF.unregister_opts(self.opts) + + def main(self, argv): # pragma: no cover + '''run the command line interface''' + try: + self._register_cli_opt() + + self._load_cli_config(argv) + + self._handle_global_opts() + + self._dispath_func_notask() + finally: + self._clear_config_opts() + + def api(self, argv, task_id): # pragma: no cover + '''run the api interface''' + try: + self._register_cli_opt() + + self._load_cli_config(argv) + + self._handle_global_opts() + + self._dispath_func_task(task_id) + finally: + self._clear_config_opts() diff --git a/yardstick/cmd/commands/env.py b/yardstick/cmd/commands/env.py new file mode 100644 index 000000000..098379ae1 --- /dev/null +++ b/yardstick/cmd/commands/env.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging + +from yardstick.common.httpClient import HttpClient +from yardstick.common import constants + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class EnvCommand(object): + ''' + + Set of commands to prepare environment + ''' + def do_influxdb(self, args): + url = constants.YARDSTICK_ENV_ACTION_API + data = {'action': 'createInfluxDBContainer'} + HttpClient().post(url, data) + logger.debug('Now creating and configing influxdb') + + def do_grafana(self, args): + url = constants.YARDSTICK_ENV_ACTION_API + data = {'action': 'createGrafanaContainer'} + HttpClient().post(url, data) + logger.debug('Now creating and configing grafana') + + def do_prepare(self, args): + url = constants.YARDSTICK_ENV_ACTION_API + data = {'action': 'prepareYardstickEnv'} + HttpClient().post(url, data) + logger.debug('Now preparing environment') diff --git a/yardstick/cmd/commands/plugin.py b/yardstick/cmd/commands/plugin.py index 9936942d8..10e5cdfbe 100644 --- a/yardstick/cmd/commands/plugin.py +++ b/yardstick/cmd/commands/plugin.py @@ -84,6 +84,7 @@ class PluginCommands(object): 'yardstick.resources', 'scripts/install/' + target_script) deployment_user = deployment.get("user") + deployment_ssh_port = deployment.get("ssh_port", ssh.DEFAULT_PORT) deployment_ip = deployment.get("ip") deployment_password = deployment.get("password") @@ -92,12 +93,14 @@ class PluginCommands(object): LOG.info("user:%s, host:%s", deployment_user, installer_ip) self.client = ssh.SSH(deployment_user, installer_ip, - password=deployment_password) + password=deployment_password, + port=deployment_ssh_port) self.client.wait(timeout=600) else: LOG.info("user:%s, host:%s", deployment_user, deployment_ip) self.client = ssh.SSH(deployment_user, deployment_ip, - password=deployment_password) + password=deployment_password, + port=deployment_ssh_port) self.client.wait(timeout=600) # copy script to host @@ -113,6 +116,7 @@ class PluginCommands(object): 'yardstick.resources', 'scripts/remove/' + target_script) deployment_user = deployment.get("user") + deployment_ssh_port = deployment.get("ssh_port", ssh.DEFAULT_PORT) deployment_ip = deployment.get("ip") deployment_password = deployment.get("password") @@ -121,12 +125,14 @@ class PluginCommands(object): LOG.info("user:%s, host:%s", deployment_user, installer_ip) self.client = ssh.SSH(deployment_user, installer_ip, - password=deployment_password) + password=deployment_password, + port=deployment_ssh_port) self.client.wait(timeout=600) else: LOG.info("user:%s, host:%s", deployment_user, deployment_ip) self.client = ssh.SSH(deployment_user, deployment_ip, - password=deployment_password) + password=deployment_password, + port=deployment_ssh_port) self.client.wait(timeout=600) # copy script to host @@ -145,6 +151,7 @@ class PluginCommands(object): class PluginParser(object): '''Parser for plugin configration files in yaml format''' + def __init__(self, path): self.path = path diff --git a/yardstick/cmd/commands/task.py b/yardstick/cmd/commands/task.py index b38e084ac..9524778ba 100644 --- a/yardstick/cmd/commands/task.py +++ b/yardstick/cmd/commands/task.py @@ -17,12 +17,15 @@ import ipaddress import time import logging import uuid +import errno from itertools import ifilter from yardstick.benchmark.contexts.base import Context from yardstick.benchmark.runners import base as base_runner from yardstick.common.task_template import TaskTemplate from yardstick.common.utils import cliargs +from yardstick.common.utils import source_env +from yardstick.common import constants output_file_default = "/tmp/yardstick.out" test_cases_dir_default = "tests/opnfv/test_cases/" @@ -51,11 +54,15 @@ class TaskCommands(object): output_file_default, default=output_file_default) @cliargs("--suite", help="process test suite file instead of a task file", action="store_true") - def do_start(self, args): + def do_start(self, args, **kwargs): '''Start a benchmark scenario.''' atexit.register(atexit_handler) + self.task_id = kwargs.get('task_id', str(uuid.uuid4())) + + check_environment() + total_start_time = time.time() parser = TaskParser(args.inputfile[0]) @@ -81,7 +88,7 @@ class TaskCommands(object): one_task_start_time = time.time() parser.path = task_files[i] scenarios, run_in_parallel, meet_precondition = parser.parse_task( - task_args[i], task_args_fnames[i]) + self.task_id, task_args[i], task_args_fnames[i]) if not meet_precondition: LOG.info("meet_precondition is %s, please check envrionment", @@ -232,7 +239,7 @@ class TaskParser(object): return valid_task_files, valid_task_args, valid_task_args_fnames - def parse_task(self, task_args=None, task_args_file=None): + def parse_task(self, task_id, task_args=None, task_args_file=None): '''parses the task file and return an context and scenario instances''' print "Parsing task config:", self.path @@ -291,7 +298,6 @@ class TaskParser(object): run_in_parallel = cfg.get("run_in_parallel", False) # add tc and task id for influxdb extended tags - task_id = str(uuid.uuid4()) for scenario in cfg["scenarios"]: task_name = os.path.splitext(os.path.basename(self.path))[0] scenario["tc"] = task_name @@ -482,3 +488,14 @@ def parse_task_args(src_name, args): % {"src": src_name, "src_type": type(kw)}) raise TypeError() return kw + + +def check_environment(): + auth_url = os.environ.get('OS_AUTH_URL', None) + if not auth_url: + try: + source_env(constants.OPENSTACK_RC_FILE) + except IOError as e: + if e.errno != errno.EEXIST: + raise + LOG.debug('OPENRC file not found') diff --git a/yardstick/cmd/commands/testcase.py b/yardstick/cmd/commands/testcase.py index 5205eb93e..cb76c7ae3 100644 --- a/yardstick/cmd/commands/testcase.py +++ b/yardstick/cmd/commands/testcase.py @@ -8,13 +8,15 @@ ############################################################################## """ Handler for yardstick command 'testcase' """ -from yardstick.cmd import print_hbar -from yardstick.common.task_template import TaskTemplate -from yardstick.common.utils import cliargs import os import yaml import sys +from yardstick.cmd import print_hbar +from yardstick.common.task_template import TaskTemplate +from yardstick.common.utils import cliargs +from yardstick.definitions import YARDSTICK_ROOT_PATH + class TestcaseCommands(object): '''Testcase commands. @@ -22,7 +24,7 @@ class TestcaseCommands(object): Set of commands to discover and display test cases. ''' def __init__(self): - self.test_case_path = 'tests/opnfv/test_cases/' + self.test_case_path = YARDSTICK_ROOT_PATH + 'tests/opnfv/test_cases/' self.testcase_list = [] def do_list(self, args): diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py new file mode 100644 index 000000000..443b3e810 --- /dev/null +++ b/yardstick/common/constants.py @@ -0,0 +1,38 @@ +import os + +DOCKER_URL = 'unix://var/run/docker.sock' + +# database config +USER = 'root' +PASSWORD = 'root' +DATABASE = 'yardstick' + +INFLUXDB_IMAGE = 'tutum/influxdb' +INFLUXDB_TAG = '0.13' + +GRAFANA_IMAGE = 'grafana/grafana' +GRAFANA_TAGS = '3.1.1' + +dirname = os.path.dirname +abspath = os.path.abspath +sep = os.path.sep + +INSTALLERS = ['apex', 'compass', 'fuel', 'joid'] + +YARDSTICK_ROOT_PATH = dirname(dirname(dirname(abspath(__file__)))) + sep + +YARDSTICK_REPOS_DIR = '/home/opnfv/repos/yardstick' + +YARDSTICK_CONFIG_DIR = '/etc/yardstick/' + +YARDSTICK_CONFIG_FILE = os.path.join(YARDSTICK_CONFIG_DIR, 'yardstick.conf') + +RELENG_DIR = '/home/opnfv/repos/releng' + +OS_FETCH_SCRIPT = 'utils/fetch_os_creds.sh' + +LOAD_IMAGES_SCRIPT = 'tests/ci/load_images.sh' + +OPENSTACK_RC_FILE = os.path.join(YARDSTICK_CONFIG_DIR, 'openstack.creds') + +YARDSTICK_ENV_ACTION_API = 'http://localhost:5000/yardstick/env/action' diff --git a/yardstick/common/httpClient.py b/yardstick/common/httpClient.py new file mode 100644 index 000000000..ab2e9a379 --- /dev/null +++ b/yardstick/common/httpClient.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import json +import logging + +import requests + +logger = logging.getLogger(__name__) + + +class HttpClient(object): + + def post(self, url, data): + data = json.dumps(data) + headers = {'Content-Type': 'application/json'} + try: + response = requests.post(url, data=data, headers=headers) + result = response.json() + logger.debug('The result is: %s', result) + + return result + except Exception as e: + logger.debug('Failed: %s', e) + raise diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index c482b4da4..3ecb0ae20 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -17,10 +17,21 @@ import os import sys +import yaml +import errno +import subprocess +import logging + from oslo_utils import importutils +from keystoneauth1 import identity +from keystoneauth1 import session +from neutronclient.v2_0 import client import yardstick +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + # Decorator for cli-args def cliargs(*args, **kwargs): @@ -68,3 +79,65 @@ def import_modules_from_package(package): new_package = ".".join(root.split(os.sep)).split("....")[1] module_name = "%s.%s" % (new_package, filename[:-3]) try_append_module(module_name, sys.modules) + + +def get_para_from_yaml(file_path, args): + + def func(a, b): + if a is None: + return None + return a.get(b) + + if os.path.exists(file_path): + with open(file_path) as f: + value = yaml.safe_load(f) + value = reduce(func, args.split('.'), value) + + if value is None: + print 'parameter not found' + return None + + return value + else: + print 'file not exist' + return None + + +def makedirs(d): + try: + os.makedirs(d) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def execute_command(cmd): + exec_msg = "Executing command: '%s'" % cmd + logger.debug(exec_msg) + + output = subprocess.check_output(cmd.split()).split(os.linesep) + + return output + + +def source_env(env_file): + p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE, + shell=True) + output = p.communicate()[0] + env = dict((line.split('=', 1) for line in output.splitlines())) + os.environ.update(env) + return env + + +def get_openstack_session(): + auth = identity.Password(auth_url=os.environ.get('OS_AUTH_URL'), + username=os.environ.get('OS_USERNAME'), + password=os.environ.get('OS_PASSWORD'), + tenant_name=os.environ.get('OS_TENANT_NAME')) + return session.Session(auth=auth) + + +def get_neutron_client(): + sess = get_openstack_session() + neutron_client = client.Client(session=sess) + return neutron_client diff --git a/yardstick/definitions.py b/yardstick/definitions.py new file mode 100644 index 000000000..300a78e58 --- /dev/null +++ b/yardstick/definitions.py @@ -0,0 +1,5 @@ +import os + +dirname = os.path.dirname +YARDSTICK_ROOT_PATH = dirname(dirname(os.path.abspath(__file__))) +YARDSTICK_ROOT_PATH += os.path.sep diff --git a/yardstick/dispatcher/file.py b/yardstick/dispatcher/file.py index ab67796e9..c2cc265ba 100644 --- a/yardstick/dispatcher/file.py +++ b/yardstick/dispatcher/file.py @@ -17,6 +17,7 @@ # ceilometer/ceilometer/dispatcher/file.py import logging +import logging.handlers import json from oslo_config import cfg diff --git a/yardstick/dispatcher/http.py b/yardstick/dispatcher/http.py index 2298d00cc..98e772dd8 100644 --- a/yardstick/dispatcher/http.py +++ b/yardstick/dispatcher/http.py @@ -81,14 +81,14 @@ class HttpDispatcher(DispatchBase): case_name = v["scenario_cfg"]["tc"] break if case_name == "": - LOG.error('Test result : %s' % json.dumps(self.result)) + LOG.error('Test result : %s', json.dumps(self.result)) LOG.error('The case_name cannot be found, no data will be posted.') return self.result["case_name"] = case_name try: - LOG.debug('Test result : %s' % json.dumps(self.result)) + LOG.debug('Test result : %s', json.dumps(self.result)) res = requests.post(self.target, data=json.dumps(self.result), headers=self.headers, diff --git a/yardstick/dispatcher/influxdb.py b/yardstick/dispatcher/influxdb.py index a9825fa35..fc9f3e932 100644 --- a/yardstick/dispatcher/influxdb.py +++ b/yardstick/dispatcher/influxdb.py @@ -16,7 +16,7 @@ import time from oslo_config import cfg from yardstick.dispatcher.base import Base as DispatchBase -from yardstick.dispatcher.influxdb_line_protocol import make_lines +from third_party.influxdb.influxdb_line_protocol import make_lines LOG = logging.getLogger(__name__) @@ -68,7 +68,9 @@ class InfluxdbDispatcher(DispatchBase): "pod_name": os.environ.get('NODE_NAME', 'unknown'), "installer": os.environ.get('INSTALLER_TYPE', 'unknown'), "deploy_scenario": os.environ.get('DEPLOY_SCENARIO', 'unknown'), - "version": os.environ.get('YARDSTICK_VERSION', 'unknown') + "version": os.path.basename(os.environ.get('YARDSTICK_BRANCH', + 'unknown')) + } def _dict_key_flatten(self, data): @@ -125,7 +127,7 @@ class InfluxdbDispatcher(DispatchBase): return make_lines(msg).encode('utf-8') def record_result_data(self, data): - LOG.debug('Test result : %s' % json.dumps(data)) + LOG.debug('Test result : %s', json.dumps(data)) self.raw_result.append(data) if self.target == '': # if the target was not set, do not do anything @@ -146,13 +148,13 @@ class InfluxdbDispatcher(DispatchBase): return 0 if self.tc == "": - LOG.error('Test result : %s' % json.dumps(data)) + LOG.error('Test result : %s', json.dumps(data)) LOG.error('The case_name cannot be found, no data will be posted.') return -1 try: line = self._data_to_line_protocol(data) - LOG.debug('Test result line format : %s' % line) + LOG.debug('Test result line format : %s', line) res = requests.post(self.influxdb_url, data=line, auth=(self.username, self.password), @@ -169,5 +171,5 @@ class InfluxdbDispatcher(DispatchBase): return 0 def flush_result_data(self): - LOG.debug('Test result all : %s' % json.dumps(self.raw_result)) + LOG.debug('Test result all : %s', json.dumps(self.raw_result)) return 0 diff --git a/yardstick/resources/files/yardstick_key b/yardstick/resources/files/yardstick_key deleted file mode 100644 index 32e860f3f..000000000 --- a/yardstick/resources/files/yardstick_key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA9EZF31uqZLHdXGZl7r12RfzJaqqt2oSBnBqFCzgve1zKbtL3 -cTKXFMHY8BqjMBF01cnx4nbtJffWy6jgqUlCgpm1sdzjSftZKhceB8LChFi4sg2K -rLjKw3mU9XhYwuWrwqE3KyMNsKuTWgW9NJQxmoDWTnqKWi+WCGuPj4sxGNt/nIq9 -uA+uzVNtRYNyRPdgCHhpTWuI+ui92vpS9IWau+A4pZqeNsJuBrG6ZUTuiiX8mq2q -MB29V/k5oQowYB5OjPzifcjwK6GciTXMIzALrsYQGFkbk90nW56FEdERtlYXw4dK -jYne7zqi24Sj3miyxZs5DfvcN1nQqei20mLfOwIDAQABAoIBACdWccYoguY4ZoeM -zfmGdVeL//u3hMvd7ulus+I8qBjbtpXmT4bhOMdU+FSiVYlWJlSPcu6fbE1i/ipK -BfP9IkLZ8hK0mb2+RnuqwWFKkfyyNPwnhh+Omsij+cMWIGUyi1iKkdHWkUvUNaSX -rAKdoudYvCpjPYiMhULR34qkRcHUtsswOeRHvxC8CXqk3fJJ/oLqCz2E4gNJs4v9 -aadxNu51ooK+srb2FcJ1zItg+NaG+Yp7aPbz+n1byH46lM4S2n9RkaoXPxUAmW9z -RxHUDDQJ6d2hP6lNDlo21Z0vINazUOjUycZ8iS7vusA6vxkfIHhmX/4XGy0/1kEs -JiUxEmECgYEA+wTqwY/LSCpcGkq+VUWG64c812ogPiRI+Wa/zJvZPL4wve3uy2e2 -Cx5auwwedracdigYr3jl85TSrXEhm2rWMkUhek+IY4jEzl2RH6/BeMenl5+Fh2Qw -ZGg7Rukn60WVArgi3KH2ipzW33YZEb3cGLHSFPG79w31Aa0mqsBZVJMCgYEA+R8Y -akSn3gOTevxczBl97BDdTndZ23+NHk52cj3TKwMnVHsAWF8ozBG7S/gjHY6Ongms -z/erBMT8yDJbSeS/SqeFnPhuBoq2CIebAc5RLK9gHDZizf/r1gin5Fyl5jbTn5em -JiOyYqVS43bAVpsJoNT9efdOFBzzNFqSOv9527kCgYEAmJ8huTSbrbILs/S0Cxat -9PCSHoupNP9M208M2PP9PoCJFEHhigzx04rOMaIpt5ZKRVEVyULh1ZssCUaa32sy -9vevZjWLQLF8r9iWD0UGhlAmZvsX7f0Nq07wk6nZmqQA+NlKYQmc5CR+RPoCPhZJ -Bz6+8/sShSEYUb+cnf87kT8CgYAPTcO4M4OEdf/HXF1vBFnh+J8/xME2ZL2MkRFh -rz6bs9PksrGwvBfLgYNaBWJS3IESYFHHbNWKs3c77SwCfBTsRyJEJFbN/BN2rq3t -DHmcHyHuWcD0GraoLVvzAWYHoHKbqTtBuIuq17Eh3BewulF7GdqAdZrMTYL7Ql0d -VrhrsQKBgBT3TOSJqc+idx58sMfZI/18GEI5PIkOuDtzgtqdwjUsIHaVnI0bVuzo -tiEl8Ma36ZsDD5JLRUF90ckeMtjHawE4KimyO51dnE4AXsMACfbdtDc5KO+BNKJz -6qw2SjRD7zlD6JYPVRERNFLGMIWLJbmD9tkswjuIOG/9ctWUC5FC ------END RSA PRIVATE KEY----- diff --git a/yardstick/resources/files/yardstick_key.pub b/yardstick/resources/files/yardstick_key.pub deleted file mode 100644 index e0d0eb580..000000000 --- a/yardstick/resources/files/yardstick_key.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD0RkXfW6pksd1cZmXuvXZF/Mlqqq3ahIGcGoULOC97XMpu0vdxMpcUwdjwGqMwEXTVyfHidu0l99bLqOCpSUKCmbWx3ONJ+1kqFx4HwsKEWLiyDYqsuMrDeZT1eFjC5avCoTcrIw2wq5NaBb00lDGagNZOeopaL5YIa4+PizEY23+cir24D67NU21Fg3JE92AIeGlNa4j66L3a+lL0hZq74Dilmp42wm4GsbplRO6KJfyaraowHb1X+TmhCjBgHk6M/OJ9yPAroZyJNcwjMAuuxhAYWRuT3SdbnoUR0RG2VhfDh0qNid7vOqLbhKPeaLLFmzkN+9w3WdCp6LbSYt87 yardstick@yardstick.opnfv.org diff --git a/yardstick/ssh.py b/yardstick/ssh.py index cf890df6f..46d53b7d2 100644 --- a/yardstick/ssh.py +++ b/yardstick/ssh.py @@ -29,24 +29,29 @@ Execute command and get output: Execute command with huge output: - class PseudoFile(object): + class PseudoFile(io.RawIOBase): def write(chunk): if "error" in chunk: email_admin(chunk) - ssh = sshclient.SSH("root", "example.com") - ssh.run("tail -f /var/log/syslog", stdout=PseudoFile(), timeout=False) + ssh = SSH("root", "example.com") + with PseudoFile() as p: + ssh.run("tail -f /var/log/syslog", stdout=p, timeout=False) Execute local script on remote side: ssh = sshclient.SSH("user", "example.com") - status, out, err = ssh.execute("/bin/sh -s arg1 arg2", - stdin=open("~/myscript.sh", "r")) + + with open("~/myscript.sh", "r") as stdin_file: + status, out, err = ssh.execute('/bin/sh -s "arg1" "arg2"', + stdin=stdin_file) Upload file: - ssh = sshclient.SSH("user", "example.com") - ssh.run("cat > ~/upload/file.gz", stdin=open("/store/file.gz", "rb")) + ssh = SSH("user", "example.com") + # use rb for binary files + with open("/store/file.gz", "rb") as stdin_file: + ssh.run("cat > ~/upload/file.gz", stdin=stdin_file) Eventlet: @@ -54,20 +59,21 @@ Eventlet: or eventlet.monkey_patch() or - sshclient = eventlet.import_patched("opentstack.common.sshclient") + sshclient = eventlet.import_patched("yardstick.ssh") """ - +import os import select import socket import time +import logging import paramiko from scp import SCPClient import six -import logging -LOG = logging.getLogger(__name__) + +DEFAULT_PORT = 22 class SSHError(Exception): @@ -81,8 +87,8 @@ class SSHTimeout(SSHError): class SSH(object): """Represent ssh connection.""" - def __init__(self, user, host, port=22, pkey=None, - key_filename=None, password=None): + def __init__(self, user, host, port=DEFAULT_PORT, pkey=None, + key_filename=None, password=None, name=None): """Initialize SSH client. :param user: ssh username @@ -92,14 +98,27 @@ class SSH(object): :param key_filename: private key filename :param password: password """ + self.name = name + if name: + self.log = logging.getLogger(__name__ + '.' + self.name) + else: + self.log = logging.getLogger(__name__) self.user = user self.host = host - self.port = port + # we may get text port from YAML, convert to int + self.port = int(port) self.pkey = self._get_pkey(pkey) if pkey else None self.password = password self.key_filename = key_filename self._client = False + # paramiko loglevel debug will output ssh protocl debug + # we don't ever really want that unless we are debugging paramiko + # ssh issues + if os.environ.get("PARAMIKO_DEBUG", "").lower() == "true": + logging.getLogger("paramiko").setLevel(logging.DEBUG) + else: + logging.getLogger("paramiko").setLevel(logging.WARN) def _get_pkey(self, key): if isinstance(key, six.string_types): @@ -137,10 +156,12 @@ class SSH(object): self._client = False def run(self, cmd, stdin=None, stdout=None, stderr=None, - raise_on_error=True, timeout=3600): + raise_on_error=True, timeout=3600, + keep_stdin_open=False): """Execute specified command on the server. :param cmd: Command to be executed. + :type cmd: str :param stdin: Open file or string to pass to stdin. :param stdout: Open file to connect to stdout. :param stderr: Open file to connect to stderr. @@ -148,6 +169,8 @@ class SSH(object): then exception will be raized if non-zero code. :param timeout: Timeout in seconds for command execution. Default 1 hour. No timeout if set to 0. + :param keep_stdin_open: don't close stdin on empty reads + :type keep_stdin_open: bool """ client = self._get_client() @@ -157,10 +180,12 @@ class SSH(object): return self._run(client, cmd, stdin=stdin, stdout=stdout, stderr=stderr, raise_on_error=raise_on_error, - timeout=timeout) + timeout=timeout, + keep_stdin_open=keep_stdin_open) def _run(self, client, cmd, stdin=None, stdout=None, stderr=None, - raise_on_error=True, timeout=3600): + raise_on_error=True, timeout=3600, + keep_stdin_open=False): transport = client.get_transport() session = transport.open_session() @@ -183,14 +208,14 @@ class SSH(object): if session.recv_ready(): data = session.recv(4096) - LOG.debug("stdout: %r" % data) + self.log.debug("stdout: %r", data) if stdout is not None: stdout.write(data) continue if session.recv_stderr_ready(): stderr_data = session.recv_stderr(4096) - LOG.debug("stderr: %r" % stderr_data) + self.log.debug("stderr: %r", stderr_data) if stderr is not None: stderr.write(stderr_data) continue @@ -200,13 +225,15 @@ class SSH(object): if not data_to_send: data_to_send = stdin.read(4096) if not data_to_send: - stdin.close() - session.shutdown_write() - writes = [] - continue - sent_bytes = session.send(data_to_send) - # LOG.debug("sent: %s" % data_to_send[:sent_bytes]) - data_to_send = data_to_send[sent_bytes:] + # we may need to keep stdin open + if not keep_stdin_open: + stdin.close() + session.shutdown_write() + writes = [] + if data_to_send: + sent_bytes = session.send(data_to_send) + # LOG.debug("sent: %s" % data_to_send[:sent_bytes]) + data_to_send = data_to_send[sent_bytes:] if session.exit_status_ready(): break @@ -253,10 +280,10 @@ class SSH(object): try: return self.execute("uname") except (socket.error, SSHError) as e: - LOG.debug("Ssh is still unavailable: %r" % e) + self.log.debug("Ssh is still unavailable: %r", e) time.sleep(interval) if time.time() > (start_time + timeout): - raise SSHTimeout("Timeout waiting for '%s'" % self.host) + raise SSHTimeout("Timeout waiting for '%s'", self.host) def put(self, files, remote_path=b'.', recursive=False): client = self._get_client() @@ -268,3 +295,37 @@ class SSH(object): def send_command(self, command): client = self._get_client() client.exec_command(command, get_pty=True) + + def _put_file_sftp(self, localpath, remotepath, mode=None): + client = self._get_client() + + with client.open_sftp() as sftp: + sftp.put(localpath, remotepath) + if mode is None: + mode = 0o777 & os.stat(localpath).st_mode + sftp.chmod(remotepath, mode) + + def _put_file_shell(self, localpath, remotepath, mode=None): + # quote to stop wordpslit + cmd = ['cat > "%s"' % remotepath] + if mode is not None: + # use -- so no options + cmd.append('chmod -- 0%o "%s"' % (mode, remotepath)) + + with open(localpath, "rb") as localfile: + # only chmod on successful cat + cmd = "&& ".join(cmd) + self.run(cmd, stdin=localfile) + + def put_file(self, localpath, remotepath, mode=None): + """Copy specified local file to the server. + + :param localpath: Local filename. + :param remotepath: Remote filename. + :param mode: Permissions to set after upload + """ + import socket + try: + self._put_file_sftp(localpath, remotepath, mode=mode) + except (paramiko.SSHException, socket.error): + self._put_file_shell(localpath, remotepath, mode=mode) |
