diff options
| -rw-r--r-- | INFO | 13 | ||||
| -rwxr-xr-x | ci/prepare_env.sh | 19 | ||||
| -rwxr-xr-x | ci/yardstick-verify | 4 | ||||
| -rw-r--r-- | docs/release/release-notes.rst | 44 | ||||
| -rw-r--r-- | docs/results/overview.rst | 6 | ||||
| -rw-r--r-- | docs/results/results.rst | 33 | ||||
| -rwxr-xr-x | docs/userguide/01-introduction.rst | 3 | ||||
| -rw-r--r-- | docs/userguide/02-methodology.rst | 4 | ||||
| -rw-r--r-- | samples/lmbench_cache.yaml | 41 | ||||
| -rw-r--r-- | tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml | 22 | ||||
| -rw-r--r-- | tests/opnfv/test_suites/opnfv_plugfest.yaml | 22 | ||||
| -rw-r--r-- | tests/unit/benchmark/scenarios/compute/test_lmbench.py | 19 | ||||
| -rwxr-xr-x | tools/ubuntu-server-cloudimg-modify.sh | 4 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/compute/lmbench.py | 19 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash | 29 |
15 files changed, 227 insertions, 55 deletions
@@ -2,8 +2,8 @@ Project: Test framework for verifying infrastructure compliance (yardstick) Project Creation Date: April 28th, 2015 Project Category: Integration & Testing Lifecycle State: Incubation -Primary Contact: ana.cunha@ericsson.com -Project Lead: ana.cunha@ericsson.com +Primary Contact: jean.gaoliang@huawei.com +Project Lead: jean.gaoliang@huawei.com Jira Project Name: Infrastructure Verification Jira Project Prefix: Yardstick Mailing list tag: [Yardstick] @@ -11,14 +11,13 @@ IRC: #opnfv-yardstick Repository: yardstick Committers: -ana.cunha@ericsson.com -leif.y.johansson@ericsson.com -hans.feldt@ericsson.com jorgen.w.karlsson@ericsson.com houjingwen@huawei.com -trevor.cooper@intel.com wenjing_chu@dell.com liangqi1@huawei.com +jean.gaoliang@huawei.com +vincenzo.m.riccobene@intel.com Link to TSC approval: http://meetbot.opnfv.org/meetings/ -Link to approval of additional submitters: +Link to approval of additional submitters: +Link to approval of new PTL: Done via Condorcet Internet Voting Service, avaliable from Raymond Piak diff --git a/ci/prepare_env.sh b/ci/prepare_env.sh index 31a8e405b..8f9f58087 100755 --- a/ci/prepare_env.sh +++ b/ci/prepare_env.sh @@ -12,10 +12,12 @@ : ${INSTALLER_TYPE:='fuel'} : ${INSTALLER_IP:='10.20.0.2'} +: ${DEPLOY_TYPE:='bm'} # Can be any of 'bm' (Bare Metal) or 'virt' (Virtual) : ${NODE_NAME:='unknown'} : ${EXTERNAL_NETWORK:='admin_floating_net'} + # Extract network name from EXTERNAL_NETWORK # e.g. EXTERNAL_NETWORK='ext-net;flat;192.168.0.2;192.168.0.253;192.168.0.1;192.168.0.0/24' export EXTERNAL_NETWORK=$(echo $EXTERNAL_NETWORK | cut -f1 -d \;) @@ -26,19 +28,16 @@ echo "INFO: Creating openstack credentials .." # Create openstack credentials OPENRC=/home/opnfv/openrc if [ ! -f $OPENRC ]; then - - $RELENG_REPO_DIR/utils/fetch_os_creds.sh \ - -d $OPENRC \ - -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} - - # Fuel virtual need a fix - if [ "$NODE_NAME" == "ericsson-virtual1" ]; then - echo "INFO: Changing: internalURL -> publicURL in openrc" - sed -i 's/internalURL/publicURL/' $OPENRC + if [ "$DEPLOY_TYPE" == "virt" ]; then + FETCH_CRED_ARG="-v -d $OPENRC -i ${INSTALLER_TYPE} -a ${INSTALLER_IP}" + else + FETCH_CRED_ARG="-d $OPENRC -i ${INSTALLER_TYPE} -a ${INSTALLER_IP}" fi + $RELENG_REPO_DIR/utils/fetch_os_creds.sh $FETCH_CRED_ARG + fi source $OPENRC -export EXTERNAL_NETWORK INSTALLER_TYPE NODE_NAME +export EXTERNAL_NETWORK INSTALLER_TYPE DEPLOY_TYPE NODE_NAME diff --git a/ci/yardstick-verify b/ci/yardstick-verify index c7b1cefec..c83193574 100755 --- a/ci/yardstick-verify +++ b/ci/yardstick-verify @@ -144,6 +144,10 @@ create_nova_flavor() echo "========== Create nova flavor ==========" # Create the nova flavor used by some sample test cases nova flavor-create yardstick-flavor 100 512 3 1 + # DPDK-enabled OVS requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then + nova flavor-key yardstick-flavor set hw:mem_page_size=large + fi fi } diff --git a/docs/release/release-notes.rst b/docs/release/release-notes.rst index b8171a135..bc58b2134 100644 --- a/docs/release/release-notes.rst +++ b/docs/release/release-notes.rst @@ -13,9 +13,9 @@ OPNFV Brahmaputra Release Note for Yardstick .. _Yardstick: https://wiki.opnfv.org/yardstick -.. _Dashboard: http://130.211.154.108/grafana/dashboard/db/yardstick-main +.. _Dashboard: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main -.. _NFV-TST001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/ +.. _NFV-TST001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf Abstract @@ -39,6 +39,9 @@ Version History | *Date* | *Version* | *Comment* | | | | | +---------------+--------------------+---------------------------------+ +| Apr 27th,2016 | 3.0 | Brahmaputra release | +| | | | ++---------------+--------------------+---------------------------------+ | Mar 30th,2016 | 2.0 | Brahmaputra release | | | | | +---------------+--------------------+---------------------------------+ @@ -129,16 +132,16 @@ Release Data | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.2.0 | +| **Repo/tag** | yardstick/brahmaputra.3.0 | | | | +--------------------------------------+--------------------------------------+ -| **Yardstick Docker image tag** | brahmaputra.2.0 | +| **Yardstick Docker image tag** | brahmaputra.3.0 | | | | +--------------------------------------+--------------------------------------+ | **Release designation** | Brahmaputra | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Mar 30th, 2016 | +| **Release date** | Apr 27th, 2016 | | | | +--------------------------------------+--------------------------------------+ | **Purpose of the delivery** | OPNFV Brahmaputra release | @@ -152,7 +155,7 @@ Version Change Module Version Changes ~~~~~~~~~~~~~~~~~~~~~~ -This is the second tracked release of Yardstick. It is based on following +This is the third tracked release of Yardstick. It is based on following upstream versions: - OpenStack Liberty @@ -163,9 +166,20 @@ upstream versions: Document Version Changes ~~~~~~~~~~~~~~~~~~~~~~~~ -This is the second tracked version of the Yardstick framework in OPNFV. +This is the third tracked version of the Yardstick framework in OPNFV. It includes the following documentation updates: +- Yardstick User Guide: corrected faulty links + +- Yardstick Code Documentation: no changes + +- Yardstick Release Notes for Yardstick: this document + +- Test Results report for Brahmaputra testing with Yardstick: updated listed of +verified scenarios and limitations + +Documentation updates on the second tracked version: + - Yardstick User Guide: added software architecture chapter - Yardstick Code Documentation: no changes @@ -182,6 +196,10 @@ Reason for Version Feature additions ~~~~~~~~~~~~~~~~~ +No new features. + +Brahmaputra.2.0: + +----------------------------+------------------------------------------------+ | **JIRA REFERENCE** | **SLOGAN** | | | | @@ -194,6 +212,10 @@ Feature additions Corrected Faults ~~~~~~~~~~~~~~~~ +No corrected faults. + +Brahmaputra.2.0: + +----------------------------+------------------------------------------------+ | **JIRA REFERENCE** | **SLOGAN** | | | | @@ -247,22 +269,22 @@ Deliverables Software Deliverables ~~~~~~~~~~~~~~~~~~~~~ -**Yardstick framework source code <brahmaputra.2.0>** +**Yardstick framework source code <brahmaputra.3.0>** +--------------------------------------+--------------------------------------+ | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.2.0 | +| **Repo/tag** | yardstick/brahmaputra.3.0 | | | | +--------------------------------------+--------------------------------------+ -| **Yardstick Docker image tag** | brahmaputra.2.0 | +| **Yardstick Docker image tag** | brahmaputra.3.0 | | | | +--------------------------------------+--------------------------------------+ | **Release designation** | Brahmaputra | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Mar 30th, 2016 | +| **Release date** | Apr 27th, 2016 | | | | +--------------------------------------+--------------------------------------+ | **Purpose of the delivery** | OPNFV Brahmaputra release | diff --git a/docs/results/overview.rst b/docs/results/overview.rst index cb1eebb2b..7f3a34e56 100644 --- a/docs/results/overview.rst +++ b/docs/results/overview.rst @@ -16,7 +16,7 @@ Introduction Document Identifier ------------------- -This document is part of deliverables of the OPNFV release brahmaputra.2.0 +This document is part of deliverables of the OPNFV release brahmaputra.3.0 Scope ----- @@ -82,13 +82,13 @@ Document change procedures and history | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.2.0 | +| **Repo/tag** | yardstick/brahmaputra.3.0 | | | | +--------------------------------------+--------------------------------------+ | **Release designation** | Brahmaputra | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Mar 30th, 2016 | +| **Release date** | Apr 27th, 2016 | | | | +--------------------------------------+--------------------------------------+ | **Purpose of the delivery** | OPNFV Brahmaputra release test | diff --git a/docs/results/results.rst b/docs/results/results.rst index d3c23f6d6..f3831b865 100644 --- a/docs/results/results.rst +++ b/docs/results/results.rst @@ -37,16 +37,17 @@ cases suite: fuel-os-nosdn-nofeature-ha.rst fuel-os-odl_l2-nofeature-ha.rst fuel-os-onos-nofeature-ha.rst + fuel-os-nosdn-kvm-ha joid-os-odl_l2-nofeature-ha.rst Limitations ----------- -The following scenarios run at least one time Yardstick test cases suite, -partially or complete: +For the following scenarios, Yardstick generic test cases suite was executed at +least one time however less than four consecutive times, measurements +collected: - * fuel-os-odl_l2-sfc-ha * fuel-os-odl_l2-bgpvpn-ha @@ -54,31 +55,25 @@ partially or complete: * joid-os-nosdn-nofeature-ha - -Test results of executed tests are avilable in Dashboard_ and logs in Jenkins_. - - -Not executed ------------- - -For the following scenarios, Yardstick test cases were not run: + * joid-os-onos-nofeature-ha - * apex-os-odl_l2-sfc-noha +For the following scenario, Yardstick generic test cases suite was executed +four consecutive times, measurements collected; no feature test cases were +executed, therefore the feature is not verified by Yardstick: - * apex-os-odl_l3-nofeature-ha + * apex-os-odl_l2-bgpvpn-ha - * apex-os-onos-nofeature-ha - * fuel-os-nosdn-kvm-ha +For the following scenario, Yardstick generic test cases suite was executed +three consecutive times, measurements collected; no feature test cases +were executed, therefore the feature is not verified by Yardstick: - * fuel-os-nosdn-ovs-ha + * fuel-os-odl_l2-sfc-ha - * fuel-os-odl_l3-nofeature-ha - * joid-os-onos-nofeature-ha +Test results of executed tests are avilable in Dashboard_ and logs in Jenkins_. -Logs are available in Jenkins_. Feature Test Results diff --git a/docs/userguide/01-introduction.rst b/docs/userguide/01-introduction.rst index 3c6100ae7..3db6ce001 100755 --- a/docs/userguide/01-introduction.rst +++ b/docs/userguide/01-introduction.rst @@ -11,8 +11,7 @@ Introduction .. _Pharos: https://wiki.opnfv.org/pharos .. _Yardstick: https://wiki.opnfv.org/yardstick -.. _Presentation: https://wiki.opnfv.org/_media/opnfv_summit_-_yardstick_project.pdf - +.. _Presentation: https://wiki.opnfv.org/download/attachments/2925202/opnfv_summit_-_yardstick_project.pdf?version=1&modificationDate=1458848320000&api=v2 Yardstick_ is an OPNFV Project. The project's goal is to verify infrastructure compliance, from the perspective diff --git a/docs/userguide/02-methodology.rst b/docs/userguide/02-methodology.rst index 7fdb5f64f..3fa432a98 100644 --- a/docs/userguide/02-methodology.rst +++ b/docs/userguide/02-methodology.rst @@ -16,8 +16,8 @@ verifying the :term:`NFVI` from the perspective of a :term:`VNF`. ETSI-NFV ======== -.. _NFV-TST001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/ -.. _Yardsticktst: https://wiki.opnfv.org/_media/opnfv_summit_-_bridging_opnfv_and_etsi.pdf +.. _NFV-TST001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf +.. _Yardsticktst: https://wiki.opnfv.org/download/attachments/2925202/opnfv_summit_-_bridging_opnfv_and_etsi.pdf?version=1&modificationDate=1458848320000&api=v2 The document ETSI GS NFV-TST001_, "Pre-deployment Testing; Report on Validation diff --git a/samples/lmbench_cache.yaml b/samples/lmbench_cache.yaml new file mode 100644 index 000000000..7a22cf15f --- /dev/null +++ b/samples/lmbench_cache.yaml @@ -0,0 +1,41 @@ +--- +# Sample benchmark task config file +# measure memory cache latency using lmbench + +schema: "yardstick:task:0.1" + +scenarios: +- + type: Lmbench + options: + test_type: "latency_for_cache" + line_size: 128 + repetition: 1 + warmup: 0 + + host: demeter.demo + + runner: + type: Iteration + iterations: 2 + interval: 1 + + sla: + max_latency: 35 + action: monitor + +context: + name: demo + image: yardstick-trusty-server + flavor: yardstick-flavor + user: ubuntu + + servers: + demeter: + floating_ip: true + + networks: + test: + cidr: '10.0.1.0/24' + + diff --git a/tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml b/tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml new file mode 100644 index 000000000..486425f1b --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml @@ -0,0 +1,22 @@ +---
+# Huawei Shanghai bare POD6 daily task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "opnfv_huawei_daily"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+ file_name: opnfv_yardstick_tc002.yaml
+-
+ file_name: opnfv_yardstick_tc005.yaml
+-
+ file_name: opnfv_yardstick_tc010.yaml
+-
+ file_name: opnfv_yardstick_tc011.yaml
+-
+ file_name: opnfv_yardstick_tc012.yaml
+-
+ file_name: opnfv_yardstick_tc014.yaml
+-
+ file_name: opnfv_yardstick_tc037.yaml
diff --git a/tests/opnfv/test_suites/opnfv_plugfest.yaml b/tests/opnfv/test_suites/opnfv_plugfest.yaml new file mode 100644 index 000000000..fde0e831b --- /dev/null +++ b/tests/opnfv/test_suites/opnfv_plugfest.yaml @@ -0,0 +1,22 @@ +--- +# Plugfest task suite + +schema: "yardstick:suite:0.1" + +name: "opnfv_plugfest" +test_cases_dir: "tests/opnfv/test_cases/" +test_cases: +- + file_name: opnfv_yardstick_tc002.yaml +- + file_name: opnfv_yardstick_tc005.yaml +- + file_name: opnfv_yardstick_tc010.yaml +- + file_name: opnfv_yardstick_tc011.yaml +- + file_name: opnfv_yardstick_tc012.yaml +- + file_name: opnfv_yardstick_tc014.yaml +- + file_name: opnfv_yardstick_tc037.yaml diff --git a/tests/unit/benchmark/scenarios/compute/test_lmbench.py b/tests/unit/benchmark/scenarios/compute/test_lmbench.py index 1b24258b6..6be116371 100644 --- a/tests/unit/benchmark/scenarios/compute/test_lmbench.py +++ b/tests/unit/benchmark/scenarios/compute/test_lmbench.py @@ -159,6 +159,25 @@ class LmbenchTestCase(unittest.TestCase): mock_ssh.SSH().execute.return_value = (0, sample_output, '') self.assertRaises(AssertionError, l.run, self.result) + def test_successful_latency_for_cache_run_sla(self, mock_ssh): + + options = { + "test_type": "latency_for_cache", + "repetition":1, + "warmup": 0 + } + args = { + "options": options, + "sla": {"max_latency": 35} + } + l = lmbench.Lmbench(args, self.ctx) + + sample_output = "{\"L1cache\": 1.6}" + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + l.run(self.result) + expected_result = json.loads(sample_output) + self.assertEqual(self.result, expected_result) + def test_unsuccessful_script_error(self, mock_ssh): options = {"test_type": "bandwidth"} diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh index 11e6051cd..78bf098dc 100755 --- a/tools/ubuntu-server-cloudimg-modify.sh +++ b/tools/ubuntu-server-cloudimg-modify.sh @@ -35,8 +35,12 @@ grep trusty /etc/apt/sources.list && \ echo 'Acquire::ForceIPv4 "true";' > /etc/apt/apt.conf.d/99force-ipv4 # Add hostname to /etc/hosts. +# Allow console access via pwd cat <<EOF >/etc/cloud/cloud.cfg.d/10_etc_hosts.cfg manage_etc_hosts: True +password: RANDOM +chpasswd: { expire: False } +ssh_pwauth: True EOF apt-get update diff --git a/yardstick/benchmark/scenarios/compute/lmbench.py b/yardstick/benchmark/scenarios/compute/lmbench.py index e15fe7eb4..d3e802f3b 100644 --- a/yardstick/benchmark/scenarios/compute/lmbench.py +++ b/yardstick/benchmark/scenarios/compute/lmbench.py @@ -57,6 +57,7 @@ class Lmbench(base.Scenario): LATENCY_BENCHMARK_SCRIPT = "lmbench_latency_benchmark.bash" BANDWIDTH_BENCHMARK_SCRIPT = "lmbench_bandwidth_benchmark.bash" + LATENCY_CACHE_SCRIPT = "lmbench_latency_for_cache.bash" def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg @@ -71,6 +72,9 @@ class Lmbench(base.Scenario): self.latency_target_script = pkg_resources.resource_filename( "yardstick.benchmark.scenarios.compute", Lmbench.LATENCY_BENCHMARK_SCRIPT) + self.latency_for_cache_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.compute", + Lmbench.LATENCY_CACHE_SCRIPT) host = self.context_cfg["host"] user = host.get("user", "ubuntu") ip = host.get("ip", None) @@ -85,6 +89,8 @@ class Lmbench(base.Scenario): stdin=open(self.latency_target_script, 'rb')) self.client.run("cat > ~/lmbench_bandwidth.sh", stdin=open(self.bandwidth_target_script, 'rb')) + self.client.run("cat > ~/lmbench_latency_for_cache.sh", + stdin=open(self.latency_for_cache_script, 'rb')) self.setup_done = True def run(self, result): @@ -106,6 +112,11 @@ class Lmbench(base.Scenario): warmup_repetitions = options.get('warmup', 0) cmd = "sudo bash lmbench_bandwidth.sh %d %s %d" % \ (size, benchmark, warmup_repetitions) + elif test_type == 'latency_for_cache': + repetition = options.get('repetition', 1) + warmup = options.get('warmup', 0) + cmd = "sudo bash lmbench_latency_for_cache.sh %d %d" % \ + (repetition, warmup) else: raise RuntimeError("No such test_type: %s for Lmbench scenario", test_type) @@ -130,12 +141,18 @@ class Lmbench(base.Scenario): if latency > sla_max_latency: sla_error += "latency %f > sla:max_latency(%f); " \ % (latency, sla_max_latency) - else: + elif test_type == 'bandwidth': sla_min_bw = int(self.scenario_cfg['sla']['min_bandwidth']) bw = result["bandwidth(MBps)"] if bw < sla_min_bw: sla_error += "bandwidth %f < " \ "sla:min_bandwidth(%f)" % (bw, sla_min_bw) + elif test_type == 'latency_for_cache': + sla_latency = float(self.scenario_cfg['sla']['max_latency']) + cache_latency = float(result['L1cache']) + if sla_latency < cache_latency: + sla_error += "latency %f > sla:max_latency(%f); " \ + % (cache_latency, sla_latency) assert sla_error == "", sla_error diff --git a/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash b/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash new file mode 100644 index 000000000..2ed1bbe14 --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash @@ -0,0 +1,29 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Run a lmbench cache latency benchmark in a host and +# outputs in json format the array sizes in megabytes and +# load latency over all points in that array in nanosecods + +set -e + +REPETITON=$1 +WARMUP=$2 + +# write the result to stdout in json format +output_json() +{ + read DATA + echo $DATA | awk '{printf "{\"L1cache\": %s}", $5}' +} + +/usr/lib/lmbench/bin/x86_64-linux-gnu/cache -W $WARMUP -N $REPETITON 2>&1 | output_json + |