summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--INFO13
-rwxr-xr-xci/prepare_env.sh19
-rwxr-xr-xci/yardstick-verify4
-rw-r--r--docs/release/release-notes.rst44
-rw-r--r--docs/results/overview.rst6
-rw-r--r--docs/results/results.rst33
-rwxr-xr-xdocs/userguide/01-introduction.rst3
-rw-r--r--docs/userguide/02-methodology.rst4
-rw-r--r--samples/lmbench_cache.yaml41
-rw-r--r--tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml22
-rw-r--r--tests/opnfv/test_suites/opnfv_plugfest.yaml22
-rw-r--r--tests/unit/benchmark/scenarios/compute/test_lmbench.py19
-rwxr-xr-xtools/ubuntu-server-cloudimg-modify.sh4
-rw-r--r--yardstick/benchmark/scenarios/compute/lmbench.py19
-rw-r--r--yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash29
15 files changed, 227 insertions, 55 deletions
diff --git a/INFO b/INFO
index 6bcf01b5a..8dd36a690 100644
--- a/INFO
+++ b/INFO
@@ -2,8 +2,8 @@ Project: Test framework for verifying infrastructure compliance (yardstick)
Project Creation Date: April 28th, 2015
Project Category: Integration & Testing
Lifecycle State: Incubation
-Primary Contact: ana.cunha@ericsson.com
-Project Lead: ana.cunha@ericsson.com
+Primary Contact: jean.gaoliang@huawei.com
+Project Lead: jean.gaoliang@huawei.com
Jira Project Name: Infrastructure Verification
Jira Project Prefix: Yardstick
Mailing list tag: [Yardstick]
@@ -11,14 +11,13 @@ IRC: #opnfv-yardstick
Repository: yardstick
Committers:
-ana.cunha@ericsson.com
-leif.y.johansson@ericsson.com
-hans.feldt@ericsson.com
jorgen.w.karlsson@ericsson.com
houjingwen@huawei.com
-trevor.cooper@intel.com
wenjing_chu@dell.com
liangqi1@huawei.com
+jean.gaoliang@huawei.com
+vincenzo.m.riccobene@intel.com
Link to TSC approval: http://meetbot.opnfv.org/meetings/
-Link to approval of additional submitters:
+Link to approval of additional submitters:
+Link to approval of new PTL: Done via Condorcet Internet Voting Service, avaliable from Raymond Piak
diff --git a/ci/prepare_env.sh b/ci/prepare_env.sh
index 31a8e405b..8f9f58087 100755
--- a/ci/prepare_env.sh
+++ b/ci/prepare_env.sh
@@ -12,10 +12,12 @@
: ${INSTALLER_TYPE:='fuel'}
: ${INSTALLER_IP:='10.20.0.2'}
+: ${DEPLOY_TYPE:='bm'} # Can be any of 'bm' (Bare Metal) or 'virt' (Virtual)
: ${NODE_NAME:='unknown'}
: ${EXTERNAL_NETWORK:='admin_floating_net'}
+
# Extract network name from EXTERNAL_NETWORK
# e.g. EXTERNAL_NETWORK='ext-net;flat;192.168.0.2;192.168.0.253;192.168.0.1;192.168.0.0/24'
export EXTERNAL_NETWORK=$(echo $EXTERNAL_NETWORK | cut -f1 -d \;)
@@ -26,19 +28,16 @@ echo "INFO: Creating openstack credentials .."
# Create openstack credentials
OPENRC=/home/opnfv/openrc
if [ ! -f $OPENRC ]; then
-
- $RELENG_REPO_DIR/utils/fetch_os_creds.sh \
- -d $OPENRC \
- -i ${INSTALLER_TYPE} -a ${INSTALLER_IP}
-
- # Fuel virtual need a fix
- if [ "$NODE_NAME" == "ericsson-virtual1" ]; then
- echo "INFO: Changing: internalURL -> publicURL in openrc"
- sed -i 's/internalURL/publicURL/' $OPENRC
+ if [ "$DEPLOY_TYPE" == "virt" ]; then
+ FETCH_CRED_ARG="-v -d $OPENRC -i ${INSTALLER_TYPE} -a ${INSTALLER_IP}"
+ else
+ FETCH_CRED_ARG="-d $OPENRC -i ${INSTALLER_TYPE} -a ${INSTALLER_IP}"
fi
+ $RELENG_REPO_DIR/utils/fetch_os_creds.sh $FETCH_CRED_ARG
+
fi
source $OPENRC
-export EXTERNAL_NETWORK INSTALLER_TYPE NODE_NAME
+export EXTERNAL_NETWORK INSTALLER_TYPE DEPLOY_TYPE NODE_NAME
diff --git a/ci/yardstick-verify b/ci/yardstick-verify
index c7b1cefec..c83193574 100755
--- a/ci/yardstick-verify
+++ b/ci/yardstick-verify
@@ -144,6 +144,10 @@ create_nova_flavor()
echo "========== Create nova flavor =========="
# Create the nova flavor used by some sample test cases
nova flavor-create yardstick-flavor 100 512 3 1
+ # DPDK-enabled OVS requires guest memory to be backed by large pages
+ if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then
+ nova flavor-key yardstick-flavor set hw:mem_page_size=large
+ fi
fi
}
diff --git a/docs/release/release-notes.rst b/docs/release/release-notes.rst
index b8171a135..bc58b2134 100644
--- a/docs/release/release-notes.rst
+++ b/docs/release/release-notes.rst
@@ -13,9 +13,9 @@ OPNFV Brahmaputra Release Note for Yardstick
.. _Yardstick: https://wiki.opnfv.org/yardstick
-.. _Dashboard: http://130.211.154.108/grafana/dashboard/db/yardstick-main
+.. _Dashboard: http://testresults.opnfv.org/grafana/dashboard/db/yardstick-main
-.. _NFV-TST001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/
+.. _NFV-TST001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf
Abstract
@@ -39,6 +39,9 @@ Version History
| *Date* | *Version* | *Comment* |
| | | |
+---------------+--------------------+---------------------------------+
+| Apr 27th,2016 | 3.0 | Brahmaputra release |
+| | | |
++---------------+--------------------+---------------------------------+
| Mar 30th,2016 | 2.0 | Brahmaputra release |
| | | |
+---------------+--------------------+---------------------------------+
@@ -129,16 +132,16 @@ Release Data
| **Project** | Yardstick |
| | |
+--------------------------------------+--------------------------------------+
-| **Repo/tag** | yardstick/brahmaputra.2.0 |
+| **Repo/tag** | yardstick/brahmaputra.3.0 |
| | |
+--------------------------------------+--------------------------------------+
-| **Yardstick Docker image tag** | brahmaputra.2.0 |
+| **Yardstick Docker image tag** | brahmaputra.3.0 |
| | |
+--------------------------------------+--------------------------------------+
| **Release designation** | Brahmaputra |
| | |
+--------------------------------------+--------------------------------------+
-| **Release date** | Mar 30th, 2016 |
+| **Release date** | Apr 27th, 2016 |
| | |
+--------------------------------------+--------------------------------------+
| **Purpose of the delivery** | OPNFV Brahmaputra release |
@@ -152,7 +155,7 @@ Version Change
Module Version Changes
~~~~~~~~~~~~~~~~~~~~~~
-This is the second tracked release of Yardstick. It is based on following
+This is the third tracked release of Yardstick. It is based on following
upstream versions:
- OpenStack Liberty
@@ -163,9 +166,20 @@ upstream versions:
Document Version Changes
~~~~~~~~~~~~~~~~~~~~~~~~
-This is the second tracked version of the Yardstick framework in OPNFV.
+This is the third tracked version of the Yardstick framework in OPNFV.
It includes the following documentation updates:
+- Yardstick User Guide: corrected faulty links
+
+- Yardstick Code Documentation: no changes
+
+- Yardstick Release Notes for Yardstick: this document
+
+- Test Results report for Brahmaputra testing with Yardstick: updated listed of
+verified scenarios and limitations
+
+Documentation updates on the second tracked version:
+
- Yardstick User Guide: added software architecture chapter
- Yardstick Code Documentation: no changes
@@ -182,6 +196,10 @@ Reason for Version
Feature additions
~~~~~~~~~~~~~~~~~
+No new features.
+
+Brahmaputra.2.0:
+
+----------------------------+------------------------------------------------+
| **JIRA REFERENCE** | **SLOGAN** |
| | |
@@ -194,6 +212,10 @@ Feature additions
Corrected Faults
~~~~~~~~~~~~~~~~
+No corrected faults.
+
+Brahmaputra.2.0:
+
+----------------------------+------------------------------------------------+
| **JIRA REFERENCE** | **SLOGAN** |
| | |
@@ -247,22 +269,22 @@ Deliverables
Software Deliverables
~~~~~~~~~~~~~~~~~~~~~
-**Yardstick framework source code <brahmaputra.2.0>**
+**Yardstick framework source code <brahmaputra.3.0>**
+--------------------------------------+--------------------------------------+
| **Project** | Yardstick |
| | |
+--------------------------------------+--------------------------------------+
-| **Repo/tag** | yardstick/brahmaputra.2.0 |
+| **Repo/tag** | yardstick/brahmaputra.3.0 |
| | |
+--------------------------------------+--------------------------------------+
-| **Yardstick Docker image tag** | brahmaputra.2.0 |
+| **Yardstick Docker image tag** | brahmaputra.3.0 |
| | |
+--------------------------------------+--------------------------------------+
| **Release designation** | Brahmaputra |
| | |
+--------------------------------------+--------------------------------------+
-| **Release date** | Mar 30th, 2016 |
+| **Release date** | Apr 27th, 2016 |
| | |
+--------------------------------------+--------------------------------------+
| **Purpose of the delivery** | OPNFV Brahmaputra release |
diff --git a/docs/results/overview.rst b/docs/results/overview.rst
index cb1eebb2b..7f3a34e56 100644
--- a/docs/results/overview.rst
+++ b/docs/results/overview.rst
@@ -16,7 +16,7 @@ Introduction
Document Identifier
-------------------
-This document is part of deliverables of the OPNFV release brahmaputra.2.0
+This document is part of deliverables of the OPNFV release brahmaputra.3.0
Scope
-----
@@ -82,13 +82,13 @@ Document change procedures and history
| **Project** | Yardstick |
| | |
+--------------------------------------+--------------------------------------+
-| **Repo/tag** | yardstick/brahmaputra.2.0 |
+| **Repo/tag** | yardstick/brahmaputra.3.0 |
| | |
+--------------------------------------+--------------------------------------+
| **Release designation** | Brahmaputra |
| | |
+--------------------------------------+--------------------------------------+
-| **Release date** | Mar 30th, 2016 |
+| **Release date** | Apr 27th, 2016 |
| | |
+--------------------------------------+--------------------------------------+
| **Purpose of the delivery** | OPNFV Brahmaputra release test |
diff --git a/docs/results/results.rst b/docs/results/results.rst
index d3c23f6d6..f3831b865 100644
--- a/docs/results/results.rst
+++ b/docs/results/results.rst
@@ -37,16 +37,17 @@ cases suite:
fuel-os-nosdn-nofeature-ha.rst
fuel-os-odl_l2-nofeature-ha.rst
fuel-os-onos-nofeature-ha.rst
+ fuel-os-nosdn-kvm-ha
joid-os-odl_l2-nofeature-ha.rst
Limitations
-----------
-The following scenarios run at least one time Yardstick test cases suite,
-partially or complete:
+For the following scenarios, Yardstick generic test cases suite was executed at
+least one time however less than four consecutive times, measurements
+collected:
- * fuel-os-odl_l2-sfc-ha
* fuel-os-odl_l2-bgpvpn-ha
@@ -54,31 +55,25 @@ partially or complete:
* joid-os-nosdn-nofeature-ha
-
-Test results of executed tests are avilable in Dashboard_ and logs in Jenkins_.
-
-
-Not executed
-------------
-
-For the following scenarios, Yardstick test cases were not run:
+ * joid-os-onos-nofeature-ha
- * apex-os-odl_l2-sfc-noha
+For the following scenario, Yardstick generic test cases suite was executed
+four consecutive times, measurements collected; no feature test cases were
+executed, therefore the feature is not verified by Yardstick:
- * apex-os-odl_l3-nofeature-ha
+ * apex-os-odl_l2-bgpvpn-ha
- * apex-os-onos-nofeature-ha
- * fuel-os-nosdn-kvm-ha
+For the following scenario, Yardstick generic test cases suite was executed
+three consecutive times, measurements collected; no feature test cases
+were executed, therefore the feature is not verified by Yardstick:
- * fuel-os-nosdn-ovs-ha
+ * fuel-os-odl_l2-sfc-ha
- * fuel-os-odl_l3-nofeature-ha
- * joid-os-onos-nofeature-ha
+Test results of executed tests are avilable in Dashboard_ and logs in Jenkins_.
-Logs are available in Jenkins_.
Feature Test Results
diff --git a/docs/userguide/01-introduction.rst b/docs/userguide/01-introduction.rst
index 3c6100ae7..3db6ce001 100755
--- a/docs/userguide/01-introduction.rst
+++ b/docs/userguide/01-introduction.rst
@@ -11,8 +11,7 @@ Introduction
.. _Pharos: https://wiki.opnfv.org/pharos
.. _Yardstick: https://wiki.opnfv.org/yardstick
-.. _Presentation: https://wiki.opnfv.org/_media/opnfv_summit_-_yardstick_project.pdf
-
+.. _Presentation: https://wiki.opnfv.org/download/attachments/2925202/opnfv_summit_-_yardstick_project.pdf?version=1&modificationDate=1458848320000&api=v2
Yardstick_ is an OPNFV Project.
The project's goal is to verify infrastructure compliance, from the perspective
diff --git a/docs/userguide/02-methodology.rst b/docs/userguide/02-methodology.rst
index 7fdb5f64f..3fa432a98 100644
--- a/docs/userguide/02-methodology.rst
+++ b/docs/userguide/02-methodology.rst
@@ -16,8 +16,8 @@ verifying the :term:`NFVI` from the perspective of a :term:`VNF`.
ETSI-NFV
========
-.. _NFV-TST001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/
-.. _Yardsticktst: https://wiki.opnfv.org/_media/opnfv_summit_-_bridging_opnfv_and_etsi.pdf
+.. _NFV-TST001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf
+.. _Yardsticktst: https://wiki.opnfv.org/download/attachments/2925202/opnfv_summit_-_bridging_opnfv_and_etsi.pdf?version=1&modificationDate=1458848320000&api=v2
The document ETSI GS NFV-TST001_, "Pre-deployment Testing; Report on Validation
diff --git a/samples/lmbench_cache.yaml b/samples/lmbench_cache.yaml
new file mode 100644
index 000000000..7a22cf15f
--- /dev/null
+++ b/samples/lmbench_cache.yaml
@@ -0,0 +1,41 @@
+---
+# Sample benchmark task config file
+# measure memory cache latency using lmbench
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+ type: Lmbench
+ options:
+ test_type: "latency_for_cache"
+ line_size: 128
+ repetition: 1
+ warmup: 0
+
+ host: demeter.demo
+
+ runner:
+ type: Iteration
+ iterations: 2
+ interval: 1
+
+ sla:
+ max_latency: 35
+ action: monitor
+
+context:
+ name: demo
+ image: yardstick-trusty-server
+ flavor: yardstick-flavor
+ user: ubuntu
+
+ servers:
+ demeter:
+ floating_ip: true
+
+ networks:
+ test:
+ cidr: '10.0.1.0/24'
+
+
diff --git a/tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml b/tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml
new file mode 100644
index 000000000..486425f1b
--- /dev/null
+++ b/tests/opnfv/test_suites/opnfv_huawei-sh-pod6_daily.yaml
@@ -0,0 +1,22 @@
+---
+# Huawei Shanghai bare POD6 daily task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "opnfv_huawei_daily"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+ file_name: opnfv_yardstick_tc002.yaml
+-
+ file_name: opnfv_yardstick_tc005.yaml
+-
+ file_name: opnfv_yardstick_tc010.yaml
+-
+ file_name: opnfv_yardstick_tc011.yaml
+-
+ file_name: opnfv_yardstick_tc012.yaml
+-
+ file_name: opnfv_yardstick_tc014.yaml
+-
+ file_name: opnfv_yardstick_tc037.yaml
diff --git a/tests/opnfv/test_suites/opnfv_plugfest.yaml b/tests/opnfv/test_suites/opnfv_plugfest.yaml
new file mode 100644
index 000000000..fde0e831b
--- /dev/null
+++ b/tests/opnfv/test_suites/opnfv_plugfest.yaml
@@ -0,0 +1,22 @@
+---
+# Plugfest task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "opnfv_plugfest"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+ file_name: opnfv_yardstick_tc002.yaml
+-
+ file_name: opnfv_yardstick_tc005.yaml
+-
+ file_name: opnfv_yardstick_tc010.yaml
+-
+ file_name: opnfv_yardstick_tc011.yaml
+-
+ file_name: opnfv_yardstick_tc012.yaml
+-
+ file_name: opnfv_yardstick_tc014.yaml
+-
+ file_name: opnfv_yardstick_tc037.yaml
diff --git a/tests/unit/benchmark/scenarios/compute/test_lmbench.py b/tests/unit/benchmark/scenarios/compute/test_lmbench.py
index 1b24258b6..6be116371 100644
--- a/tests/unit/benchmark/scenarios/compute/test_lmbench.py
+++ b/tests/unit/benchmark/scenarios/compute/test_lmbench.py
@@ -159,6 +159,25 @@ class LmbenchTestCase(unittest.TestCase):
mock_ssh.SSH().execute.return_value = (0, sample_output, '')
self.assertRaises(AssertionError, l.run, self.result)
+ def test_successful_latency_for_cache_run_sla(self, mock_ssh):
+
+ options = {
+ "test_type": "latency_for_cache",
+ "repetition":1,
+ "warmup": 0
+ }
+ args = {
+ "options": options,
+ "sla": {"max_latency": 35}
+ }
+ l = lmbench.Lmbench(args, self.ctx)
+
+ sample_output = "{\"L1cache\": 1.6}"
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+ l.run(self.result)
+ expected_result = json.loads(sample_output)
+ self.assertEqual(self.result, expected_result)
+
def test_unsuccessful_script_error(self, mock_ssh):
options = {"test_type": "bandwidth"}
diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh
index 11e6051cd..78bf098dc 100755
--- a/tools/ubuntu-server-cloudimg-modify.sh
+++ b/tools/ubuntu-server-cloudimg-modify.sh
@@ -35,8 +35,12 @@ grep trusty /etc/apt/sources.list && \
echo 'Acquire::ForceIPv4 "true";' > /etc/apt/apt.conf.d/99force-ipv4
# Add hostname to /etc/hosts.
+# Allow console access via pwd
cat <<EOF >/etc/cloud/cloud.cfg.d/10_etc_hosts.cfg
manage_etc_hosts: True
+password: RANDOM
+chpasswd: { expire: False }
+ssh_pwauth: True
EOF
apt-get update
diff --git a/yardstick/benchmark/scenarios/compute/lmbench.py b/yardstick/benchmark/scenarios/compute/lmbench.py
index e15fe7eb4..d3e802f3b 100644
--- a/yardstick/benchmark/scenarios/compute/lmbench.py
+++ b/yardstick/benchmark/scenarios/compute/lmbench.py
@@ -57,6 +57,7 @@ class Lmbench(base.Scenario):
LATENCY_BENCHMARK_SCRIPT = "lmbench_latency_benchmark.bash"
BANDWIDTH_BENCHMARK_SCRIPT = "lmbench_bandwidth_benchmark.bash"
+ LATENCY_CACHE_SCRIPT = "lmbench_latency_for_cache.bash"
def __init__(self, scenario_cfg, context_cfg):
self.scenario_cfg = scenario_cfg
@@ -71,6 +72,9 @@ class Lmbench(base.Scenario):
self.latency_target_script = pkg_resources.resource_filename(
"yardstick.benchmark.scenarios.compute",
Lmbench.LATENCY_BENCHMARK_SCRIPT)
+ self.latency_for_cache_script = pkg_resources.resource_filename(
+ "yardstick.benchmark.scenarios.compute",
+ Lmbench.LATENCY_CACHE_SCRIPT)
host = self.context_cfg["host"]
user = host.get("user", "ubuntu")
ip = host.get("ip", None)
@@ -85,6 +89,8 @@ class Lmbench(base.Scenario):
stdin=open(self.latency_target_script, 'rb'))
self.client.run("cat > ~/lmbench_bandwidth.sh",
stdin=open(self.bandwidth_target_script, 'rb'))
+ self.client.run("cat > ~/lmbench_latency_for_cache.sh",
+ stdin=open(self.latency_for_cache_script, 'rb'))
self.setup_done = True
def run(self, result):
@@ -106,6 +112,11 @@ class Lmbench(base.Scenario):
warmup_repetitions = options.get('warmup', 0)
cmd = "sudo bash lmbench_bandwidth.sh %d %s %d" % \
(size, benchmark, warmup_repetitions)
+ elif test_type == 'latency_for_cache':
+ repetition = options.get('repetition', 1)
+ warmup = options.get('warmup', 0)
+ cmd = "sudo bash lmbench_latency_for_cache.sh %d %d" % \
+ (repetition, warmup)
else:
raise RuntimeError("No such test_type: %s for Lmbench scenario",
test_type)
@@ -130,12 +141,18 @@ class Lmbench(base.Scenario):
if latency > sla_max_latency:
sla_error += "latency %f > sla:max_latency(%f); " \
% (latency, sla_max_latency)
- else:
+ elif test_type == 'bandwidth':
sla_min_bw = int(self.scenario_cfg['sla']['min_bandwidth'])
bw = result["bandwidth(MBps)"]
if bw < sla_min_bw:
sla_error += "bandwidth %f < " \
"sla:min_bandwidth(%f)" % (bw, sla_min_bw)
+ elif test_type == 'latency_for_cache':
+ sla_latency = float(self.scenario_cfg['sla']['max_latency'])
+ cache_latency = float(result['L1cache'])
+ if sla_latency < cache_latency:
+ sla_error += "latency %f > sla:max_latency(%f); " \
+ % (cache_latency, sla_latency)
assert sla_error == "", sla_error
diff --git a/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash b/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash
new file mode 100644
index 000000000..2ed1bbe14
--- /dev/null
+++ b/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Run a lmbench cache latency benchmark in a host and
+# outputs in json format the array sizes in megabytes and
+# load latency over all points in that array in nanosecods
+
+set -e
+
+REPETITON=$1
+WARMUP=$2
+
+# write the result to stdout in json format
+output_json()
+{
+ read DATA
+ echo $DATA | awk '{printf "{\"L1cache\": %s}", $5}'
+}
+
+/usr/lib/lmbench/bin/x86_64-linux-gnu/cache -W $WARMUP -N $REPETITON 2>&1 | output_json
+