NFVBENCH-160 xtesting integration4.2.2.2

Change-Id: Id600a1277059e84bc2d46fa20294fb22729f9c1f
Signed-off-by: fmenguy <francoisregis.menguy@orange.com>

9 files changed, 302 insertions, 7 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 1cb7c71..cdd7f92 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -23,14 +23,16 @@ RUN apt-get update && apt-get install -y \
        iproute2 \
        libelf1 \
        && ln -s /usr/bin/python3.6 /usr/local/bin/python3 \
-       && mkdir -p /opt/trex \
+       && mkdir -p /opt/trex/$TREX_VER \
        && mkdir /var/log/nfvbench \
-       && wget --no-cache https://trex-tgn.cisco.com/trex/release/$TREX_VER.tar.gz \
-       && tar xzf $TREX_VER.tar.gz -C /opt/trex \
+       && mkdir /tmp/trex \
+       && wget --no-cache https://github.com/cisco-system-traffic-generator/trex-core/archive/$TREX_VER.tar.gz \
+       && tar xzf $TREX_VER.tar.gz --strip-components=1 -C /tmp/trex \
        && rm -f /$TREX_VER.tar.gz \
-       && rm -f /opt/trex/$TREX_VER/trex_client_$TREX_VER.tar.gz \
-       && cp -a /opt/trex/$TREX_VER/automation/trex_control_plane/interactive/trex /usr/local/lib/python3.6/dist-packages/ \
+       && cp -a /tmp/trex/scripts/automation/trex_control_plane/interactive/trex /usr/local/lib/python3.6/dist-packages/ \
+       && cp -a /tmp/trex/scripts/* /opt/trex/$TREX_VER/ \
        && rm -rf /opt/trex/$TREX_VER/automation/trex_control_plane/interactive/trex \
+       && rm -rf /tmp/trex \
        && wget https://bootstrap.pypa.io/get-pip.py \
        && python3 get-pip.py \
        && pip3 install -U pbr \
@@ -39,6 +41,8 @@ RUN apt-get update && apt-get install -y \
        && git clone https://gerrit.opnfv.org/gerrit/nfvbench \
        && cd /nfvbench && pip3 install -e . \
        && wget -O nfvbenchvm-$VM_IMAGE_VER.qcow2 http://artifacts.opnfv.org/nfvbench/images/nfvbenchvm_centos-$VM_IMAGE_VER.qcow2 \
+       # Override Xtesting testcases.yaml file by NFVbench default one
+       && cp xtesting/testcases.yaml /usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml \
        && python3 ./docker/cleanup_generators.py \
        && rm -rf /nfvbench/.git \
        && apt-get remove -y wget git \
diff --git a/docker/nfvbench-entrypoint.sh b/docker/nfvbench-entrypoint.sh
index a7195a3..812816d 100755
--- a/docker/nfvbench-entrypoint.sh
+++ b/docker/nfvbench-entrypoint.sh
@@ -13,9 +13,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-
-if [ -z "$1" ] ||  [ $1 != 'start_rest_server' ]; then
+if [ -z "$1" ] ||  ([ $1 != 'start_rest_server' ] &&  [ $1 != 'run_tests' ]); then
         tail -f /dev/null
+elif [ $1 == 'run_tests' ]; then
+        PARAMS=""
+        for var in "${@:2}"
+        do
+            PARAMS+="$var "
+        done
+        eval "run_tests $PARAMS"
 else
         PARAMS="--server"
         if [ -n "$HOST" ]; then
diff --git a/docs/testing/user/userguide/images/nfvbench-xtesting.png b/docs/testing/user/userguide/images/nfvbench-xtesting.png
new file mode 100644
index 0000000..d5dca8d
--- /dev/null
+++ b/docs/testing/user/userguide/images/nfvbench-xtesting.png
diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst
index a61fa46..78519c8 100644
--- a/docs/testing/user/userguide/index.rst
+++ b/docs/testing/user/userguide/index.rst
@@ -25,6 +25,7 @@ Table of Content
    examples
    advanced
    pvpl3
+   xtesting
    mpls
    extchains
    fluentd
diff --git a/docs/testing/user/userguide/xtesting.rst b/docs/testing/user/userguide/xtesting.rst
new file mode 100644
index 0000000..0b863ca
--- /dev/null
+++ b/docs/testing/user/userguide/xtesting.rst
@@ -0,0 +1,237 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. SPDX-License-Identifier: CC-BY-4.0
+
+
+NFVbench Xtesting test cases and Xtesting CI integration
+--------------------------------------------------------
+
+NFVbench can leverages on `Xtesting CI <https://galaxy.ansible.com/collivier/xtesting>`_ and the common test case execution proposed by `Xtesting <https://xtesting.readthedocs.io/en/latest/>`_.
+Thanks to a simple test case list, this tool deploys anywhere plug-and-play `CI/CD toolchains in a few commands <https://wiki.opnfv.org/pages/viewpage.action?pageId=32015004>`_.
+In addition, it supports multiple components such as Jenkins and Gitlab CI (test schedulers) and multiple deployment models such as all-in-one or centralized services.
+
+NFVbench using Xtesting and Xtesting CI will permit:
+
+- smoothly assemble multiple heterogeneous test cases
+- generate the Jenkins jobs
+- deploy local CI/CD toolchains everywhere
+- dump all test case results and logs for third-party conformance review
+
+Xtesting CI only requires GNU/Linux as Operating System and asks for a few dependencies as described in Deploy your own Xtesting CI/CD toolchains:
+
+- python-virtualenv
+- docker.io
+- git
+
+Please note the next two points depending on the GNU/Linux distributions and the network settings:
+
+- SELinux: you may have to add --system-site-packages when creating the virtualenv ("Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!")
+- Proxy: you may set your proxy in env for Ansible and in systemd for Docker https://docs.docker.com/config/daemon/systemd/#httphttps-proxy
+
+Here is the default NFVbench tree as proposed in ``xtesting/ansible/host_vars/127.0.0.1`` file:
+
+- ``/home/opnfv/nfvbench``
+
+File content:
+
+.. code-block:: bash
+
+    docker_args:
+    env: {}
+    volumes:
+      - /lib/modules/$(uname -r):/lib/modules/$(uname -r)
+      - /usr/src/kernels:/usr/src/kernels -v /dev:/dev
+      - /home/opnfv/nfvbench:/tmp/nfvbench
+
+Please note: replace ``/home/opnfv/nfvbench`` with appropriate path to permit NFVbench container to access config file
+
+To deploy your own CI toolchain running NFVbench:
+
+.. code-block:: bash
+
+    virtualenv nfvbench
+    . nfvbench/bin/activate
+    pip install ansible
+    ansible-galaxy install collivier.xtesting
+    git clone https://gerrit.opnfv.org/gerrit/nfvbench nfvbench
+    ansible-playbook nfvbench/xtesting/ansible/site.yml
+
+
+NFVbench Xtesting test cases and existing CI chain
+--------------------------------------------------
+
+For test automation purpose, Xtesting framework can be used as an executor of NFVbench test cases and called by a CI chain (Jenkins, Gitlab CI ...).
+Xtesting use a testcases.yaml file to list and run test case. One basic testcases.yaml is provided by NFVbench natively but can be override.
+
+Example of CI scenario:
+
+.. image:: images/nfvbench-xtesting.png
+
+1. Run NFVbench container using Xtesting python library
+
+The NFVbench container can be started using docker run command.
+
+To run NFVbench using docker run:
+
+.. code-block:: bash
+
+    docker run --rm \
+        -e S3_ENDPOINT_URL=http://127.0.0.1:9000 \
+        -e S3_DST_URL=s3://xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \
+        -e HTTP_DST_URL=http://127.0.0.1:8181/$BUILD_TAG/$JOB_NAME-$BUILD_ID \
+        -e AWS_ACCESS_KEY_ID=xtesting \
+        -e AWS_SECRET_ACCESS_KEY=xtesting \
+        -e TEST_DB_URL=http://127.0.0.1:8000/api/v1/results \
+        -e NODE_NAME=nfvbench \
+        -e BUILD_TAG=$BUILD_TAG \
+        --privileged \
+        -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) \
+        -v /usr/src/kernels:/usr/src/kernels -v /dev:/dev \
+        -v $HOME/nfvbench:/tmp/nfvbench \
+        -v $HOME/workspace/$JOB_NAME/results:/var/lib/xtesting/results \
+        opnfv/nfvbench run_tests -t 10kpps-pvp-run -r -p
+
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| Docker options                                                | Description                                                                |
++===============================================================+============================================================================+
+| --rm                                                          | clean up container after execution                                         |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e S3_ENDPOINT_URL                                            | (Xtesting) Environnement variable used to store NFVbench artifacts to Minio|
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e S3_DST_URL                                                 | (Xtesting) Environnement variable used for S3 storage destination          |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e HTTP_DST_URL                                               | (Xtesting) Environnement variable used for S3www service                   |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e AWS_ACCESS_KEY_ID                                          | (Xtesting) Environnement variable used for S3 access key                   |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e AWS_SECRET_ACCESS_KEY                                      | (Xtesting) Environnement variable used for S3 access secret                |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e TEST_DB_URL                                                | (Xtesting) Environnement variable used to export NFVbench results in DB    |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e NODE_NAME                                                  | (Xtesting) Environnement variable used as result key identifier in DB      |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -e BUILD_TAG                                                  | (Xtesting) Environnement variable used as result key identifier in DB      |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| --privileged                                                  | (optional) required if SELinux is enabled on the host                      |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -v /lib/modules:/lib/modules                                  | needed by kernel modules in the container                                  |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -v /usr/src/kernels:/usr/src/kernels                          | needed by TRex to build kernel modules when needed                         |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -v /dev:/dev                                                  | needed by kernel modules in the container                                  |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -v $HOME/nfvbench:/tmp/nfvbench                               | folder mapping to pass files between the                                   |
+|                                                               | host and the docker space (see examples below)                             |
+|                                                               | Here we map the $HOME/nfvbench directory on the host                       |
+|                                                               | to the /tmp/nfvbench director in the container.                            |
+|                                                               | Any other mapping can work as well                                         |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -v $HOME/workspace/$JOB_NAME/results:/var/lib/xtesting/results| (Xtesting) folder mapping to pass files between the                        |
+|                                                               | CI chain workspace and the docker space to store Xtesting result files     |
+|                                                               | in orchestrator (Jenkins, Gitlab ...)                                      |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| opnfv/nfvbench                                                | container image name                                                       |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| run_tests                                                     | (Xtesting) Xtesting command to run test cases                              |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -t 10kpps-pvp-run                                             | (Xtesting) Xtesting parameter: Test case or tier (group of tests)          |
+|                                                               | to be executed. It will run all the test if not specified.                 |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -r                                                            | (Xtesting) Xtesting parameter: publish result to database                  |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+| -p                                                            | (Xtesting) Xtesting parameter: publish artifacts to a S3 service           |
++---------------------------------------------------------------+----------------------------------------------------------------------------+
+
+2. Run Xtesting test cases
+
+Executed directly by NFVbench docker entrypoint after docker start.
+
+3. Perform NFVbench test
+
+Xtesting call NFVbench python script to execute test case scenario and wait for run to be terminated.
+
+4. Export NFVbench result
+
+If ``-r`` option is used, results are pushed to a DB through Xtesting.
+If ``-p`` option is used, results are pushed to a S3 service through Xtesting.
+
+
+Override testcases.yaml file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To replace existing testcases.yaml file, using Xtesting CI add the volume mapping in ``xtesting/ansible/host_vars/127.0.0.1`` file:
+
+.. code-block:: bash
+
+    docker_args:
+    env: {}
+    volumes:
+      - /lib/modules/$(uname -r):/lib/modules/$(uname -r)
+      - /usr/src/kernels:/usr/src/kernels -v /dev:/dev
+      - /home/opnfv/nfvbench:/tmp/nfvbench
+      - /home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml
+
+* ``/home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside.
+
+
+To replace existing testcases.yaml file, using NFVbench container:
+
+.. code-block:: bash
+
+    docker run --name nfvbench --detach --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) -v /usr/src/kernels:/usr/src/kernels -v /dev:/dev -v $HOME/nfvbench:/tmp/nfvbench \
+    -v $HOME/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml \
+    opnfv/nfvbench
+
+
+* ``$HOME/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside.
+
+
+Example of Xtesting test case
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    ---
+    tiers:
+        -
+            name: nfvbench
+            order: 1
+            description: 'Data Plane Performance Testing'
+            testcases:
+                -
+                    case_name: 10kpps-pvp-run
+                    project_name: nfvbench
+                    criteria: 100
+                    blocking: true
+                    clean_flag: false
+                    description: ''
+                    run:
+                        name: 'bashfeature'
+                        args:
+                            cmd:
+                                - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps
+
+
+Examples of manual run
+~~~~~~~~~~~~~~~~~~~~~~
+
+If NFVbench container is already started in CLI mode (see Starting NFVbench in CLI mode dedicated chapter).
+To do a single run at 10,000pps bi-directional (or 5kpps in each direction) using the PVP packet path:
+
+.. code-block:: bash
+
+   docker exec -it nfvbench run_tests -t 10kpps-pvp-run
+
+Xtesting option used:
+
+* ``-t 10kpps-pvp-run`` : specify the test case to run
+
+To pass all test cases:
+
+.. code-block:: bash
+
+   docker exec -it nfvbench run_tests -t all
+
+Xtesting option used:
+
+* ``-t all`` : select all test cases existing in testcases.yaml file
+
diff --git a/requirements.txt b/requirements.txt
index 9eb76c4..430d70c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,3 +22,4 @@ tabulate>=0.7.5
 flask>=0.12
 fluent-logger>=0.5.3
 netaddr>=0.7.19
+xtesting>=0.87.0
diff --git a/xtesting/ansible/host_vars/127.0.0.1 b/xtesting/ansible/host_vars/127.0.0.1
new file mode 100644
index 0000000..9e3f1b9
--- /dev/null
+++ b/xtesting/ansible/host_vars/127.0.0.1
@@ -0,0 +1,6 @@
+docker_args:
+  env: {}
+  volumes:
+    - /lib/modules/$(uname -r):/lib/modules/$(uname -r)
+    - /usr/src/kernels:/usr/src/kernels -v /dev:/dev
+    - /home/opnfv/nfvbench:/tmp/nfvbench
\ No newline at end of file
diff --git a/xtesting/ansible/site.yml b/xtesting/ansible/site.yml
new file mode 100644
index 0000000..4643a32
--- /dev/null
+++ b/xtesting/ansible/site.yml
@@ -0,0 +1,20 @@
+---
+- hosts:
+    - 127.0.0.1
+  roles:
+    - role: collivier.xtesting
+      project: nfvbench
+      gerrit_project: nfvbench
+      builds:
+        steps:
+          - name: build opnfv/nfvbench
+            containers:
+              - name: nfvbench
+                ref_arg: BRANCH
+                path: docker
+      suites:
+        - container: nfvbench
+          tests:
+            - 10kpps-pvp-run
+          properties:
+            execution-type: SEQUENTIALLY
diff --git a/xtesting/testcases.yaml b/xtesting/testcases.yaml
new file mode 100644
index 0000000..cbb5c45
--- /dev/null
+++ b/xtesting/testcases.yaml
@@ -0,0 +1,20 @@
+---
+tiers:
+    -
+        name: nfvbench
+        order: 1
+        description: 'Data Plane Performance Testing'
+        testcases:
+            -
+                case_name: 10kpps-pvp-run
+                project_name: nfvbench
+                criteria: 100
+                blocking: true
+                clean_flag: false
+                description: ''
+                run:
+                    name: 'bashfeature'
+                    args:
+                        cmd:
+                            - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps
+