diff options
| -rw-r--r-- | docs/release/release-notes/release_notes.rst | 100 | ||||
| -rw-r--r-- | docs/release/release-notes/requirements_notes.rst | 13 | ||||
| -rw-r--r-- | docs/testing/developer/deprecated.rst | 15 | ||||
| -rw-r--r-- | docs/testing/developer/index.rst | 6 | ||||
| -rw-r--r-- | docs/testing/developer/overview.rst | 2 | ||||
| -rw-r--r-- | docs/testing/user/userguide/deprecated.rst | 28 | ||||
| -rw-r--r-- | docs/testing/user/userguide/index.rst | 18 | ||||
| -rw-r--r-- | docs/testing/user/userguide/posca/posca_stress_ping.rst | 34 | ||||
| -rw-r--r-- | docs/testing/user/userguide/posca/posca_stress_traffic.rst | 32 | ||||
| -rwxr-xr-x | run_tests.sh | 100 | ||||
| -rw-r--r-- | testsuites/posca/testcase_cfg/posca_factor_ping.yaml (renamed from testsuites/posca/testcase_cfg/posca_stress_ping.yaml) | 0 | ||||
| -rw-r--r-- | testsuites/posca/testcase_script/posca_factor_ping.py (renamed from testsuites/posca/testcase_script/posca_stress_ping.py) | 0 | ||||
| -rw-r--r-- | testsuites/posca/testsuite_story/posca_factor_test | 5 |
13 files changed, 231 insertions, 122 deletions
diff --git a/docs/release/release-notes/release_notes.rst b/docs/release/release-notes/release_notes.rst index 76b5c1cf..dd08a0e4 100644 --- a/docs/release/release-notes/release_notes.rst +++ b/docs/release/release-notes/release_notes.rst @@ -38,33 +38,27 @@ Version History | Sept 22nd, 2016| 1.0 | Bottlenecks Colorado release 1.0| | | | | +----------------+--------------------+---------------------------------+ -| Feb 17nd, 2016 | 1.1 | Bottlenecks Danube release 1.0 | +| Feb 17nd, 2017 | 1.1 | Bottlenecks Danube release 1.0 | +| | | | ++----------------+--------------------+---------------------------------+ +| Mar 24nd, 2017 | 1.2 | Bottlenecks Danube release 1.0 | | | | | +----------------+--------------------+---------------------------------+ - Summary ======= * Documentation generated with Sphinx - * User guide - - * Installation Procedure - - * Release Notes (this document) + * Release - * Platform Overview + * Release Notes - * Configuration Guide + * Testing -* Bottlenecks test suite + * User Guide - * Jenkins Jobs for OPNFV community labs - -* Bottlenecks framework - -* Bottlenecks test cases + * Development Guide The *Bottlenecks software* is developed in the OPNFV community, by the Bottlenecks_ team. @@ -72,26 +66,26 @@ Bottlenecks_ team. Release Data ============ -Bramaputra Release Data +Danube Release Data ----------------------- +--------------------------------------+--------------------------------------+ | **Project** | Bottlenecks | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | Bottlenecks/brahmaputra.1.0 | +| **Repo/tag** | Bottlenecks/danube.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Bottlenecks Docker image tag** | brahmaputra.1.0 | +| **Bottlenecks Docker image tag** | danube.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Brahmaputra base release | +| **Release designation** | Danube 1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | February 25 2016 | +| **Release date** | March 31 2017 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | Brahmaputra base release | +| **Purpose of the delivery** | Danube base release | | | | +--------------------------------------+--------------------------------------+ @@ -108,7 +102,7 @@ Colorado Release Data | **Bottlenecks Docker image tag** | colorado.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Colorado base release | +| **Release designation** | Colorado 1.0 | | | | +--------------------------------------+--------------------------------------+ | **Release date** | September 22 2016 | @@ -118,9 +112,29 @@ Colorado Release Data | | | +--------------------------------------+--------------------------------------+ -Colorado Release Data +Bramaputra Release Data ----------------------- -* TODO + ++--------------------------------------+--------------------------------------+ +| **Project** | Bottlenecks | +| | | ++--------------------------------------+--------------------------------------+ +| **Repo/tag** | Bottlenecks/brahmaputra.1.0 | +| | | ++--------------------------------------+--------------------------------------+ +| **Bottlenecks Docker image tag** | brahmaputra.1.0 | +| | | ++--------------------------------------+--------------------------------------+ +| **Release designation** | Brahmaputra 1.0 | +| | | ++--------------------------------------+--------------------------------------+ +| **Release date** | February 25 2016 | +| | | ++--------------------------------------+--------------------------------------+ +| **Purpose of the delivery** | Brahmaputra base release | +| | | ++--------------------------------------+--------------------------------------+ + Danube Deliverables =================== @@ -138,11 +152,9 @@ Documentatiion Deliverables **Bottlenecks documentation <danube.1.0>** -* Platformation Overview: http://artifacts.opnfv.org/bottlenecks/danube/1.0/docs/platformoverview/index.html -* Configuration Guide: http://artifacts.opnfv.org/bottlenecks/danube/1.0/configurationguide/index.html -* Installation Guide: http://artifacts.opnfv.org/bottlenecks/danube/docs/installationprocedure/index.html -* http://artifacts.opnfv.org/bottlenecks/danube/1.0/releasenotes/index.html -* User Guide: http://artifacts.opnfv.org/bottlenecks/danube/1.0/docs/userguide/index.html +* Release Notes: TO-DO +* User Guide: TO-DO +* Development Guide: TO-DO Colorado Deliverables ===================== @@ -169,7 +181,8 @@ Documentatiion Deliverables Reason for Version ================== -* In Danube, POSCA testsuite is introduced and two stress tests are implemented. +* In Danube, POSCA testsuite is introduced and two stress tests are implemented. Rubbos and vstf are +not supported in this release. * In Colorado, rubbos is refactored by using puppet, which makes it quite flexible to configure with different number of load generator(Client), worker(tomcat). * In Colorado, vstf is refactored by extracting the test case's configuration information. @@ -187,6 +200,27 @@ Test results are available in: - jenkins logs on CI: https://build.opnfv.org/ci/view/bottlenecks/ +The test results are reported to MongoDB. An example is given below. + +:: + + { + "project_name": "bottlenecks", + "scenario": "os-odl_l2-nofeature-ha", + "stop_date": null, + "trust_indicator": null, + "case_name": "posca_stress_ping", + "build_tag": "bottlenecks-compass-posca_stress_ping-baremetal-daily-master", + "version": "master", + "pod_name": "huawei-pod2", + "criteria": "PASS", + "installer": "compass", + "_id": "58cf4d3e32c829000a1150a8", + "start_date": "2017-3-9 4:33:04", + "details": {} + } + +For more information, please refer to: https://wiki.opnfv.org/display/testing/Result+alignment+for+ELK+post-processing Open JIRA tickets ================= @@ -194,10 +228,12 @@ Open JIRA tickets +------------------+-----------------------------------------------+ | JIRA | Description | +==================+===============================================+ +| BOTTLENECK-103 | Refactoring the Bottlenecks tesing workflow | +------------------+-----------------------------------------------+ +| BOTTLENECK-135 | Avoiding docker-compose building new image | +| | for Bottlenecks | +------------------+-----------------------------------------------+ -+------------------+-----------------------------------------------+ -+------------------+-----------------------------------------------+ +| BOTTLENECK-147 | Investigting why calling remote docker client | +------------------+-----------------------------------------------+ diff --git a/docs/release/release-notes/requirements_notes.rst b/docs/release/release-notes/requirements_notes.rst index 2660bb01..9e1c4719 100644 --- a/docs/release/release-notes/requirements_notes.rst +++ b/docs/release/release-notes/requirements_notes.rst @@ -11,13 +11,14 @@ Bottlenecks Requirements Notes for OPNFV Danube Release Installer Requirements ====================== -* TODO +In Danube, test cases are runned defaultly over compass4nfv deployment. ===================== Hardware Requirements ===================== -* TODO +No specific requirements for hardware, as long as there is external network connection. +So that the dependent package could be downloaded. ===================== Software Requirements @@ -26,7 +27,12 @@ Software Requirements Operation System ---------------- -* TODO +Ubuntu 14.04 is the default system. However, we do not do any linux core operation or call any system related api directly. The software could be runned in higher versions of Ubuntu with high probability. Thorough tests are not performed. + +Cloud OS +-------- + +Bottlenecks support OpenStack Newton with compatibility for both openstack V2/V3 API. That is, Bottlenecks could be runned over lower version of OpenStack. Depending Package ----------------- @@ -103,3 +109,4 @@ Depending Package - wrapt==1.10.6 - pyroute2==0.4.10 - elasticsearch==5.0.1 +- docker==2.0.2 diff --git a/docs/testing/developer/deprecated.rst b/docs/testing/developer/deprecated.rst new file mode 100644 index 00000000..1566f38c --- /dev/null +++ b/docs/testing/developer/deprecated.rst @@ -0,0 +1,15 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) Huawei Technologies Co.,Ltd and others. + +************************************** +Bottlenecks - Deprecated Testing Guide +************************************** + + +.. toctree:: + :maxdepth: 1 + + ./rubbos_guide.rst + ./vstf_guide.rst + diff --git a/docs/testing/developer/index.rst b/docs/testing/developer/index.rst index 20cc00eb..8d57ead0 100644 --- a/docs/testing/developer/index.rst +++ b/docs/testing/developer/index.rst @@ -7,10 +7,10 @@ Bottlenecks - Testing Guide *************************** .. toctree:: - :maxdepth: 2 + :maxdepth: 1 ./overview.rst ./posca_guide.rst - ./rubbos_guide.rst - ./vstf_guide.rst ./dashboard_guide.rst + ./deprecated.rst + diff --git a/docs/testing/developer/overview.rst b/docs/testing/developer/overview.rst index 8bc1b7c1..42f06eb7 100644 --- a/docs/testing/developer/overview.rst +++ b/docs/testing/developer/overview.rst @@ -25,7 +25,7 @@ vstf is refactored by extracting the test case's configuration information. Danube: posca testsuite is introduced to implementing stress (factor), scenario and tuning test in parametric manner. Two testcases are developed and integrated into -community CI pipeline. +community CI pipeline. Rubbos and VSTF are not supported any more. Integration Description ======================= diff --git a/docs/testing/user/userguide/deprecated.rst b/docs/testing/user/userguide/deprecated.rst new file mode 100644 index 00000000..80fe44af --- /dev/null +++ b/docs/testing/user/userguide/deprecated.rst @@ -0,0 +1,28 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +*********************************** +Bottlenecks - Deprecated Test Cases +*********************************** + +.. This index.rst file is used to structure and organise the content in these directories. +.. The documentation toolchain will create web and pdf versions of the content of this file. + +.. toctree:: + :maxdepth: 1 + + ./rubbos/rubbos_basic.rst + ./rubbos/rubbos_tc1101.rst + ./rubbos/rubbos_tc1201.rst + ./rubbos/rubbos_tc1301.rst + ./rubbos/rubbos_tc1401.rst + ./rubbos/rubbos_heavy_tc1101.rst + ./vstf/vstf_Ti1.rst + ./vstf/vstf_Ti2.rst + ./vstf/vstf_Ti3.rst + ./vstf/vstf_Tn1.rst + ./vstf/vstf_Tn2.rst + ./vstf/vstf_Tu1.rst + ./vstf/vstf_Tu2.rst + ./vstf/vstf_Tu3.rst + diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 8afd57ab..795530d6 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -9,21 +9,9 @@ Bottlenecks - User Guide .. The documentation toolchain will create web and pdf versions of the content of this file. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - ./rubbos/rubbos_basic.rst - ./rubbos/rubbos_tc1101.rst - ./rubbos/rubbos_tc1201.rst - ./rubbos/rubbos_tc1301.rst - ./rubbos/rubbos_tc1401.rst - ./rubbos/rubbos_heavy_tc1101.rst - ./vstf/vstf_Ti1.rst - ./vstf/vstf_Ti2.rst - ./vstf/vstf_Ti3.rst - ./vstf/vstf_Tn1.rst - ./vstf/vstf_Tn2.rst - ./vstf/vstf_Tu1.rst - ./vstf/vstf_Tu2.rst - ./vstf/vstf_Tu3.rst ./posca/posca_stress_ping.rst ./posca/posca_stress_traffic.rst + ./deprecated.rst + diff --git a/docs/testing/user/userguide/posca/posca_stress_ping.rst b/docs/testing/user/userguide/posca/posca_stress_ping.rst index 846b4686..963cf508 100644 --- a/docs/testing/user/userguide/posca/posca_stress_ping.rst +++ b/docs/testing/user/userguide/posca/posca_stress_ping.rst @@ -32,21 +32,23 @@ Test Case Configuration ============ -load_manager: - scenarios: - tool: ping - test_times: 100 - package_size: - num_stack: 2, 5, 10 - package_loss: 10% +:: - contexts: - stack_create: yardstick - flavor: - yardstick_test_ip: - yardstick_test_dir: "samples" - yardstick_testcase: "ping_bottlenecks" + load_manager: + scenarios: + tool: ping + test_times: 100 + package_size: + num_stack: 2, 5, 10 + package_loss: 10% -dashboard: - dashboard: "y" - dashboard_ip: + contexts: + stack_create: yardstick + flavor: + yardstick_test_ip: + yardstick_test_dir: "samples" + yardstick_testcase: "ping_bottlenecks" + + dashboard: + dashboard: "y" + dashboard_ip: diff --git a/docs/testing/user/userguide/posca/posca_stress_traffic.rst b/docs/testing/user/userguide/posca/posca_stress_traffic.rst index a995996e..5f02e036 100644 --- a/docs/testing/user/userguide/posca/posca_stress_traffic.rst +++ b/docs/testing/user/userguide/posca/posca_stress_traffic.rst @@ -32,18 +32,20 @@ Test Case Configration =========== -test_config: - tool: netperf - protocol: tcp - test_time: 20 - tx_pkt_sizes: 64, 256, 1024, 4096, 8192, 16384, 32768, 65536 - rx_pkt_sizes: 64, 256, 1024, 4096, 8192, 16384, 32768, 65536 - cpu_load: 0.9 - latency: 100000 -runner_config: - dashboard: "y" - dashboard_ip: - stack_create: yardstick - yardstick_test_ip: - yardstick_test_dir: "samples" - yardstick_testcase: "netperf_bottlenecks" +:: + + test_config: + tool: netperf + protocol: tcp + test_time: 20 + tx_pkt_sizes: 64, 256, 1024, 4096, 8192, 16384, 32768, 65536 + rx_pkt_sizes: 64, 256, 1024, 4096, 8192, 16384, 32768, 65536 + cpu_load: 0.9 + latency: 100000 + runner_config: + dashboard: "y" + dashboard_ip: + stack_create: yardstick + yardstick_test_ip: + yardstick_test_dir: "samples" + yardstick_testcase: "netperf_bottlenecks" diff --git a/run_tests.sh b/run_tests.sh index 798a5bac..f6744bf1 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -11,55 +11,64 @@ usage="Script to run the tests in bottlenecks auto. usage: - bash $(basename "$0") [-h|--help] [-s <test suite>] + bash $(basename "$0") [-h|--help] [-s <test suite>] [-c <test case>] where: -h|--help show the help text -r|--report push results to DB(true by default) - -s|--suite run specific test suite - <test suite> one of the following: - rubbos, vstf, posca + -s|--teststory run specific test story + <test story> one of the following: + (rubbos, vstf, posca_factor_test) + user can also define their own test story and pass as var to this file, + please refer to testsuites/posca/testsuite_story/ for details + -c|--testcase run specific test case + <test case> one of the following: + (posca_factor_system_bandwidth, posca_factor_ping) examples: $(basename "$0") - $(basename "$0") -s posca" + $(basename "$0") -s posca_factor_test" + +Bottlenecks_key_dir="/home/opnfv/bottlenecks/utils/infra_setup" +POSCA_SCRIPT="/home/opnfv/bottlenecks/testsuites/posca" +SUITE_PREFIX="/home/opnfv/bottlenecks/testsuites/posca/testcase_cfg" report=true -arr_test_suite=(rubbos vstf posca) +#TO-DO add auto-find for test story as for test case +all_test_story=(rubbos vstf posca_factor_test) -Bottlenecks_key_dir="/home/opnfv/bottlenecks/utils/infra_setup" +find $SUITE_PREFIX -name "*yaml" > /tmp/all_testcases.yaml +all_testcases_posca=`cat /tmp/all_testcases.yaml | awk -F '/' '{print $NF}' | awk -F '.' '{print $1}'` +all_test_case=(${all_testcases_posca}) function run_test(){ - test_suite=$1 - echo "Running test suite $test_suite" + test_exec=$1 - case $test_suite in + case $test_exec in "rubbos") info "After OPNFV Colorado release, Rubbos testsuite is not updating anymore. -This entrance for running Rubbos within Bottlenecks is no longer supported. -This testsuite is also not in the release plan with Bottlenecks since then. -If you want to run Rubbos, please refer to earlier releases.\n" + This entrance for running Rubbos within Bottlenecks is no longer supported. + This testsuite is also not in the release plan with Bottlenecks since then. + If you want to run Rubbos, please refer to earlier releases." ;; "vstf") info "After OPNFV Colorado release, VSTF testsuite is not updating anymore. -This entrance for running VSTF within Bottlenecks is no longer supported. -This testsuite is also not in the release plan with Bottlenecks since then. -If you want to run VSTF, please refer to earlier releases.\n" + This entrance for running VSTF within Bottlenecks is no longer supported. + This testsuite is also not in the release plan with Bottlenecks since then. + If you want to run VSTF, please refer to earlier releases." ;; - "posca") - POSCA_SCRIPT=/home/opnfv/bottlenecks/testsuites/posca - TEST_CASE=posca_factor_system_bandwidth + *) info "Composing up dockers" docker-compose -f /home/opnfv/bottlenecks/docker/bottleneck-compose/docker-compose.yml up -d info "Pulling tutum/influxdb for yardstick" docker pull tutum/influxdb:0.13 - info "Copying testing scripts to docker" - docker cp /home/opnfv/bottlenecks/run_posca.sh bottleneckcompose_bottlenecks_1:/home/opnfv/bottlenecks +# info "Copying testing scripts to docker" +# docker cp /home/opnfv/bottlenecks/run_posca.sh bottleneckcompose_bottlenecks_1:/home/opnfv/bottlenecks sleep 5 - info "Running posca test suite with default testcase posca_stress_traffic" - docker exec bottleneckcompose_bottlenecks_1 python ${POSCA_SCRIPT}/run_posca.py testcase $TEST_CASE + info "Running posca test story: $test_exec" + docker exec bottleneckcompose_bottlenecks_1 python ${POSCA_SCRIPT}/run_posca.py $test_level $test_exec ;; esac } @@ -76,8 +85,12 @@ while [[ $# > 0 ]] -r|--report) report="-r" ;; - -s|--suite) - SUITE="$2" + -s|--teststory) + teststory="$2" + shift + ;; + -c|--testcase) + testcase="$2" shift ;; *) @@ -104,23 +117,38 @@ ssh-keygen -t rsa -f $Bottlenecks_key_dir/bottlenecks_key/bottlenecks_key -q -N chmod 600 $Bottlenecks_key_dir/bottlenecks_key/* #check the test suite name is correct -if [ "${SUITE}" != "" ]; then - suite_exec=(${SUITE//,/ }) - for i in "${suite_exec[@]}"; do - if [[ " ${arr_test_suite[*]} " != *" $i "* ]]; then - error "unkown test suite: $i" +if [ "${teststory}" != "" ]; then + teststory_exec=(${teststory//,/ }) + for i in "${teststory_exec[@]}"; do + if [[ " ${all_test_story[*]} " != *" $i "* ]]; then + error "Unkown test story: $i" fi done - info "Tests to execute: ${SUITE}" fi -# Source credentials -info "Sourcing Credentials openstack.creds to run the tests.." -source /home/opnfv/bottlenecks/config/openstack.creds +#check the test case name is correct +if [ "${testcase}" != "" ]; then + testcase_exec=(${testcase//,/ }) + for i in "${testcase_exec[@]}"; do + if [[ " ${all_test_case[*]} " != *" $i "* ]]; then + error "Unkown test case: $i" + fi + done +fi #run tests -if [ "${SUITE}" != "" ]; then - for i in "${suite_exec[@]}"; do +if [ "${teststory}" != "" ]; then + test_level="teststory" + for i in "${teststory_exec[@]}"; do + info "Start to run test story $i" run_test $i done fi + +if [ "${testcase}" != "" ]; then + test_level="testcase" + for i in "${testcase_exec[@]}"; do + info "Start to run test case $i" + run_test $i + done +fi
\ No newline at end of file diff --git a/testsuites/posca/testcase_cfg/posca_stress_ping.yaml b/testsuites/posca/testcase_cfg/posca_factor_ping.yaml index 7e6bf027..7e6bf027 100644 --- a/testsuites/posca/testcase_cfg/posca_stress_ping.yaml +++ b/testsuites/posca/testcase_cfg/posca_factor_ping.yaml diff --git a/testsuites/posca/testcase_script/posca_stress_ping.py b/testsuites/posca/testcase_script/posca_factor_ping.py index 732422ea..732422ea 100644 --- a/testsuites/posca/testcase_script/posca_stress_ping.py +++ b/testsuites/posca/testcase_script/posca_factor_ping.py diff --git a/testsuites/posca/testsuite_story/posca_factor_test b/testsuites/posca/testsuite_story/posca_factor_test index a19fce59..9e9909e2 100644 --- a/testsuites/posca/testsuite_story/posca_factor_test +++ b/testsuites/posca/testsuite_story/posca_factor_test @@ -1 +1,4 @@ -posca_factor_system_bandwidth
\ No newline at end of file +testcase: +- + posca_factor_system_bandwidth + |