diff options
| -rw-r--r-- | docker/Dockerfile | 14 | ||||
| -rwxr-xr-x | docker/nfvbench-entrypoint.sh | 10 | ||||
| -rw-r--r-- | docs/testing/user/userguide/images/nfvbench-xtesting.png | bin | 0 -> 17342 bytes | |||
| -rw-r--r-- | docs/testing/user/userguide/index.rst | 1 | ||||
| -rw-r--r-- | docs/testing/user/userguide/xtesting.rst | 237 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | xtesting/ansible/host_vars/127.0.0.1 | 6 | ||||
| -rw-r--r-- | xtesting/ansible/site.yml | 20 | ||||
| -rw-r--r-- | xtesting/testcases.yaml | 20 |
9 files changed, 302 insertions, 7 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index 1cb7c71..cdd7f92 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,14 +23,16 @@ RUN apt-get update && apt-get install -y \ iproute2 \ libelf1 \ && ln -s /usr/bin/python3.6 /usr/local/bin/python3 \ - && mkdir -p /opt/trex \ + && mkdir -p /opt/trex/$TREX_VER \ && mkdir /var/log/nfvbench \ - && wget --no-cache https://trex-tgn.cisco.com/trex/release/$TREX_VER.tar.gz \ - && tar xzf $TREX_VER.tar.gz -C /opt/trex \ + && mkdir /tmp/trex \ + && wget --no-cache https://github.com/cisco-system-traffic-generator/trex-core/archive/$TREX_VER.tar.gz \ + && tar xzf $TREX_VER.tar.gz --strip-components=1 -C /tmp/trex \ && rm -f /$TREX_VER.tar.gz \ - && rm -f /opt/trex/$TREX_VER/trex_client_$TREX_VER.tar.gz \ - && cp -a /opt/trex/$TREX_VER/automation/trex_control_plane/interactive/trex /usr/local/lib/python3.6/dist-packages/ \ + && cp -a /tmp/trex/scripts/automation/trex_control_plane/interactive/trex /usr/local/lib/python3.6/dist-packages/ \ + && cp -a /tmp/trex/scripts/* /opt/trex/$TREX_VER/ \ && rm -rf /opt/trex/$TREX_VER/automation/trex_control_plane/interactive/trex \ + && rm -rf /tmp/trex \ && wget https://bootstrap.pypa.io/get-pip.py \ && python3 get-pip.py \ && pip3 install -U pbr \ @@ -39,6 +41,8 @@ RUN apt-get update && apt-get install -y \ && git clone https://gerrit.opnfv.org/gerrit/nfvbench \ && cd /nfvbench && pip3 install -e . \ && wget -O nfvbenchvm-$VM_IMAGE_VER.qcow2 http://artifacts.opnfv.org/nfvbench/images/nfvbenchvm_centos-$VM_IMAGE_VER.qcow2 \ + # Override Xtesting testcases.yaml file by NFVbench default one + && cp xtesting/testcases.yaml /usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml \ && python3 ./docker/cleanup_generators.py \ && rm -rf /nfvbench/.git \ && apt-get remove -y wget git \ diff --git a/docker/nfvbench-entrypoint.sh b/docker/nfvbench-entrypoint.sh index a7195a3..812816d 100755 --- a/docker/nfvbench-entrypoint.sh +++ b/docker/nfvbench-entrypoint.sh @@ -13,9 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. # - -if [ -z "$1" ] || [ $1 != 'start_rest_server' ]; then +if [ -z "$1" ] || ([ $1 != 'start_rest_server' ] && [ $1 != 'run_tests' ]); then tail -f /dev/null +elif [ $1 == 'run_tests' ]; then + PARAMS="" + for var in "${@:2}" + do + PARAMS+="$var " + done + eval "run_tests $PARAMS" else PARAMS="--server" if [ -n "$HOST" ]; then diff --git a/docs/testing/user/userguide/images/nfvbench-xtesting.png b/docs/testing/user/userguide/images/nfvbench-xtesting.png Binary files differnew file mode 100644 index 0000000..d5dca8d --- /dev/null +++ b/docs/testing/user/userguide/images/nfvbench-xtesting.png diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index a61fa46..78519c8 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -25,6 +25,7 @@ Table of Content examples advanced pvpl3 + xtesting mpls extchains fluentd diff --git a/docs/testing/user/userguide/xtesting.rst b/docs/testing/user/userguide/xtesting.rst new file mode 100644 index 0000000..0b863ca --- /dev/null +++ b/docs/testing/user/userguide/xtesting.rst @@ -0,0 +1,237 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + + +NFVbench Xtesting test cases and Xtesting CI integration +-------------------------------------------------------- + +NFVbench can leverages on `Xtesting CI <https://galaxy.ansible.com/collivier/xtesting>`_ and the common test case execution proposed by `Xtesting <https://xtesting.readthedocs.io/en/latest/>`_. +Thanks to a simple test case list, this tool deploys anywhere plug-and-play `CI/CD toolchains in a few commands <https://wiki.opnfv.org/pages/viewpage.action?pageId=32015004>`_. +In addition, it supports multiple components such as Jenkins and Gitlab CI (test schedulers) and multiple deployment models such as all-in-one or centralized services. + +NFVbench using Xtesting and Xtesting CI will permit: + +- smoothly assemble multiple heterogeneous test cases +- generate the Jenkins jobs +- deploy local CI/CD toolchains everywhere +- dump all test case results and logs for third-party conformance review + +Xtesting CI only requires GNU/Linux as Operating System and asks for a few dependencies as described in Deploy your own Xtesting CI/CD toolchains: + +- python-virtualenv +- docker.io +- git + +Please note the next two points depending on the GNU/Linux distributions and the network settings: + +- SELinux: you may have to add --system-site-packages when creating the virtualenv ("Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!") +- Proxy: you may set your proxy in env for Ansible and in systemd for Docker https://docs.docker.com/config/daemon/systemd/#httphttps-proxy + +Here is the default NFVbench tree as proposed in ``xtesting/ansible/host_vars/127.0.0.1`` file: + +- ``/home/opnfv/nfvbench`` + +File content: + +.. code-block:: bash + + docker_args: + env: {} + volumes: + - /lib/modules/$(uname -r):/lib/modules/$(uname -r) + - /usr/src/kernels:/usr/src/kernels -v /dev:/dev + - /home/opnfv/nfvbench:/tmp/nfvbench + +Please note: replace ``/home/opnfv/nfvbench`` with appropriate path to permit NFVbench container to access config file + +To deploy your own CI toolchain running NFVbench: + +.. code-block:: bash + + virtualenv nfvbench + . nfvbench/bin/activate + pip install ansible + ansible-galaxy install collivier.xtesting + git clone https://gerrit.opnfv.org/gerrit/nfvbench nfvbench + ansible-playbook nfvbench/xtesting/ansible/site.yml + + +NFVbench Xtesting test cases and existing CI chain +-------------------------------------------------- + +For test automation purpose, Xtesting framework can be used as an executor of NFVbench test cases and called by a CI chain (Jenkins, Gitlab CI ...). +Xtesting use a testcases.yaml file to list and run test case. One basic testcases.yaml is provided by NFVbench natively but can be override. + +Example of CI scenario: + +.. image:: images/nfvbench-xtesting.png + +1. Run NFVbench container using Xtesting python library + +The NFVbench container can be started using docker run command. + +To run NFVbench using docker run: + +.. code-block:: bash + + docker run --rm \ + -e S3_ENDPOINT_URL=http://127.0.0.1:9000 \ + -e S3_DST_URL=s3://xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ + -e HTTP_DST_URL=http://127.0.0.1:8181/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ + -e AWS_ACCESS_KEY_ID=xtesting \ + -e AWS_SECRET_ACCESS_KEY=xtesting \ + -e TEST_DB_URL=http://127.0.0.1:8000/api/v1/results \ + -e NODE_NAME=nfvbench \ + -e BUILD_TAG=$BUILD_TAG \ + --privileged \ + -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) \ + -v /usr/src/kernels:/usr/src/kernels -v /dev:/dev \ + -v $HOME/nfvbench:/tmp/nfvbench \ + -v $HOME/workspace/$JOB_NAME/results:/var/lib/xtesting/results \ + opnfv/nfvbench run_tests -t 10kpps-pvp-run -r -p + ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| Docker options | Description | ++===============================================================+============================================================================+ +| --rm | clean up container after execution | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e S3_ENDPOINT_URL | (Xtesting) Environnement variable used to store NFVbench artifacts to Minio| ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e S3_DST_URL | (Xtesting) Environnement variable used for S3 storage destination | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e HTTP_DST_URL | (Xtesting) Environnement variable used for S3www service | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e AWS_ACCESS_KEY_ID | (Xtesting) Environnement variable used for S3 access key | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e AWS_SECRET_ACCESS_KEY | (Xtesting) Environnement variable used for S3 access secret | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e TEST_DB_URL | (Xtesting) Environnement variable used to export NFVbench results in DB | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e NODE_NAME | (Xtesting) Environnement variable used as result key identifier in DB | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e BUILD_TAG | (Xtesting) Environnement variable used as result key identifier in DB | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| --privileged | (optional) required if SELinux is enabled on the host | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v /lib/modules:/lib/modules | needed by kernel modules in the container | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v /usr/src/kernels:/usr/src/kernels | needed by TRex to build kernel modules when needed | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v /dev:/dev | needed by kernel modules in the container | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v $HOME/nfvbench:/tmp/nfvbench | folder mapping to pass files between the | +| | host and the docker space (see examples below) | +| | Here we map the $HOME/nfvbench directory on the host | +| | to the /tmp/nfvbench director in the container. | +| | Any other mapping can work as well | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v $HOME/workspace/$JOB_NAME/results:/var/lib/xtesting/results| (Xtesting) folder mapping to pass files between the | +| | CI chain workspace and the docker space to store Xtesting result files | +| | in orchestrator (Jenkins, Gitlab ...) | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| opnfv/nfvbench | container image name | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| run_tests | (Xtesting) Xtesting command to run test cases | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -t 10kpps-pvp-run | (Xtesting) Xtesting parameter: Test case or tier (group of tests) | +| | to be executed. It will run all the test if not specified. | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -r | (Xtesting) Xtesting parameter: publish result to database | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -p | (Xtesting) Xtesting parameter: publish artifacts to a S3 service | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ + +2. Run Xtesting test cases + +Executed directly by NFVbench docker entrypoint after docker start. + +3. Perform NFVbench test + +Xtesting call NFVbench python script to execute test case scenario and wait for run to be terminated. + +4. Export NFVbench result + +If ``-r`` option is used, results are pushed to a DB through Xtesting. +If ``-p`` option is used, results are pushed to a S3 service through Xtesting. + + +Override testcases.yaml file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To replace existing testcases.yaml file, using Xtesting CI add the volume mapping in ``xtesting/ansible/host_vars/127.0.0.1`` file: + +.. code-block:: bash + + docker_args: + env: {} + volumes: + - /lib/modules/$(uname -r):/lib/modules/$(uname -r) + - /usr/src/kernels:/usr/src/kernels -v /dev:/dev + - /home/opnfv/nfvbench:/tmp/nfvbench + - /home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml + +* ``/home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside. + + +To replace existing testcases.yaml file, using NFVbench container: + +.. code-block:: bash + + docker run --name nfvbench --detach --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) -v /usr/src/kernels:/usr/src/kernels -v /dev:/dev -v $HOME/nfvbench:/tmp/nfvbench \ + -v $HOME/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml \ + opnfv/nfvbench + + +* ``$HOME/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside. + + +Example of Xtesting test case +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + --- + tiers: + - + name: nfvbench + order: 1 + description: 'Data Plane Performance Testing' + testcases: + - + case_name: 10kpps-pvp-run + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'bashfeature' + args: + cmd: + - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps + + +Examples of manual run +~~~~~~~~~~~~~~~~~~~~~~ + +If NFVbench container is already started in CLI mode (see Starting NFVbench in CLI mode dedicated chapter). +To do a single run at 10,000pps bi-directional (or 5kpps in each direction) using the PVP packet path: + +.. code-block:: bash + + docker exec -it nfvbench run_tests -t 10kpps-pvp-run + +Xtesting option used: + +* ``-t 10kpps-pvp-run`` : specify the test case to run + +To pass all test cases: + +.. code-block:: bash + + docker exec -it nfvbench run_tests -t all + +Xtesting option used: + +* ``-t all`` : select all test cases existing in testcases.yaml file + diff --git a/requirements.txt b/requirements.txt index 9eb76c4..430d70c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ tabulate>=0.7.5 flask>=0.12 fluent-logger>=0.5.3 netaddr>=0.7.19 +xtesting>=0.87.0 diff --git a/xtesting/ansible/host_vars/127.0.0.1 b/xtesting/ansible/host_vars/127.0.0.1 new file mode 100644 index 0000000..9e3f1b9 --- /dev/null +++ b/xtesting/ansible/host_vars/127.0.0.1 @@ -0,0 +1,6 @@ +docker_args: + env: {} + volumes: + - /lib/modules/$(uname -r):/lib/modules/$(uname -r) + - /usr/src/kernels:/usr/src/kernels -v /dev:/dev + - /home/opnfv/nfvbench:/tmp/nfvbench
\ No newline at end of file diff --git a/xtesting/ansible/site.yml b/xtesting/ansible/site.yml new file mode 100644 index 0000000..4643a32 --- /dev/null +++ b/xtesting/ansible/site.yml @@ -0,0 +1,20 @@ +--- +- hosts: + - 127.0.0.1 + roles: + - role: collivier.xtesting + project: nfvbench + gerrit_project: nfvbench + builds: + steps: + - name: build opnfv/nfvbench + containers: + - name: nfvbench + ref_arg: BRANCH + path: docker + suites: + - container: nfvbench + tests: + - 10kpps-pvp-run + properties: + execution-type: SEQUENTIALLY diff --git a/xtesting/testcases.yaml b/xtesting/testcases.yaml new file mode 100644 index 0000000..cbb5c45 --- /dev/null +++ b/xtesting/testcases.yaml @@ -0,0 +1,20 @@ +--- +tiers: + - + name: nfvbench + order: 1 + description: 'Data Plane Performance Testing' + testcases: + - + case_name: 10kpps-pvp-run + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'bashfeature' + args: + cmd: + - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps + |
