Add xrally_kubernetes_full

It iterates 10 times all operations run by xrally_kubernetes.
Now times, concurrenty and count can be set via testcases.yaml.

It creates a new containers functest-kubernetes-benchamrking as
Functest.

Change-Id: Ibfd493b987a359a3f16c2d8db0aef6b8e90b7e92
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit a0f76d884c1f13c27e906136d3418f12b5342fc4)

10 files changed, 149 insertions, 64 deletions

diff --git a/.travis.yml b/.travis.yml
index 641e408e..a494f2c0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -146,3 +146,29 @@ jobs:
           --platforms linux/amd64,linux/arm,linux/arm64 \
           --template ${DOCKER_USERNAME}/functest-kubernetes-smoke:ARCH-iruya \
           --target ${DOCKER_USERNAME}/functest-kubernetes-smoke:iruya
+    - stage: build functest-kubernetes-benchmarking image
+      script: bash build.sh
+      env:
+        - REPO="${DOCKER_USERNAME}"
+        - amd64_dirs="docker/benchmarking"
+        - arm_dirs=""
+        - arm64_dirs=""
+    - script: bash build.sh
+      env:
+        - REPO="${DOCKER_USERNAME}"
+        - amd64_dirs=""
+        - arm_dirs="docker/benchmarking"
+        - arm64_dirs=""
+    - script: bash build.sh
+      env:
+        - REPO="${DOCKER_USERNAME}"
+        - amd64_dirs=""
+        - arm_dirs=""
+        - arm64_dirs="docker/benchmarking"
+    - stage: publish functest-kubernetes-benchmarking manifests
+      script: >
+        manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm,linux/arm64 \
+          --template \
+        ${DOCKER_USERNAME}/functest-kubernetes-benchmarking:ARCH-iruya \
+          --target ${DOCKER_USERNAME}/functest-kubernetes-benchmarking:iruya
diff --git a/ansible/site.yml b/ansible/site.yml
index 9586f9f6..b628fce1 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -20,7 +20,7 @@
               - name: functest-kubernetes-core
                 ref_arg: BRANCH
                 path: docker/core
-          - name: build opnfv/functest-kubernetes-healthcheck
+          - name: build containers
             containers:
               - name: functest-kubernetes-healthcheck
                 ref_arg:
@@ -31,11 +31,16 @@
               - name: functest-kubernetes-security
                 ref_arg: BRANCH
                 path: docker/security
-          - name: build containers
+          - name: build opnfv/functest-kubernetes-smoke
             containers:
               - name: functest-kubernetes-smoke
                 ref_arg:
                 path: docker/smoke
+          - name: build opnfv/functest-kubernetes-benchmarking
+            containers:
+              - name: functest-kubernetes-benchmarking
+                ref_arg:
+                path: docker/benchmarking
       suites:
         - container: functest-kubernetes-healthcheck
           tests:
@@ -48,6 +53,9 @@
           tests:
             - kube_hunter
             - kube_bench
+        - container: functest-kubernetes-benchmarking
+          tests:
+            - xrally_kubernetes_full
         - container: functest-kubernetes-cnf
           tests:
             - k8s_vims
diff --git a/build.sh b/build.sh
index 64b5b49c..d51f9319 100644
--- a/build.sh
+++ b/build.sh
@@ -8,7 +8,8 @@ docker/core \
 docker/healthcheck \
 docker/smoke \
 docker/cnf \
-docker/security"}
+docker/security \
+docker/benchmarking"}
 arm_dirs=${arm_dirs-${amd64_dirs}}
 arm64_dirs=${arm64_dirs-${amd64_dirs}}
 build_opts=(--pull=true --no-cache --force-rm=true)
@@ -42,6 +43,9 @@ ${repo}/functest-kubernetes-core:arm64-iruya|g" {} +
 find . -name Dockerfile -exec sed -i \
     -e "s|opnfv/functest-kubernetes-healthcheck:iruya|\
 ${repo}/functest-kubernetes-healthcheck:arm64-iruya|g" {} +
+find . -name Dockerfile -exec sed -i \
+    -e "s|opnfv/functest-kubernetes-smoke:iruya|\
+${repo}/functest-kubernetes-smoke:arm64-iruya|g" {} +
 for dir in ${arm64_dirs}; do
     (cd "${dir}" && docker build "${build_opts[@]}" \
         -t "${repo}/functest-kubernetes-${dir##**/}:arm64-iruya" .)
@@ -63,6 +67,9 @@ ${repo}/functest-kubernetes-core:arm-iruya|g" {} +
 find . -name Dockerfile -exec sed -i \
     -e "s|opnfv/functest-kubernetes-healthcheck:iruya|\
 ${repo}/functest-kubernetes-healthcheck:arm-iruya|g" {} +
+find . -name Dockerfile -exec sed -i \
+    -e "s|opnfv/functest-kubernetes-smoke:iruya|\
+${repo}/functest-kubernetes-smoke:arm-iruya|g" {} +
 for dir in ${arm_dirs}; do
     (cd "${dir}" && docker build "${build_opts[@]}" \
         -t "${repo}/functest-kubernetes-${dir##**/}:arm-iruya" .)
diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile
new file mode 100644
index 00000000..42ea0bda
--- /dev/null
+++ b/docker/benchmarking/Dockerfile
@@ -0,0 +1,4 @@
+FROM opnfv/functest-kubernetes-smoke:iruya
+
+COPY testcases.yaml /usr/lib/python3.6/site-packages/xtesting/ci/testcases.yaml
+CMD ["run_tests", "-t", "all"]
diff --git a/docker/benchmarking/hooks/post_checkout b/docker/benchmarking/hooks/post_checkout
new file mode 100644
index 00000000..6598f040
--- /dev/null
+++ b/docker/benchmarking/hooks/post_checkout
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+from="${DOCKER_REPO%/*}/functest-kubernetes-smoke:${DOCKER_TAG}"
+sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile
+
+exit $?
diff --git a/docker/benchmarking/testcases.yaml b/docker/benchmarking/testcases.yaml
new file mode 100644
index 00000000..f4800095
--- /dev/null
+++ b/docker/benchmarking/testcases.yaml
@@ -0,0 +1,25 @@
+---
+tiers:
+    -
+        name: benchmarking
+        order: 1
+        description: >-
+            A set of benchmarking tests integrated from xRally for Kubernetes
+            platform.
+        testcases:
+            -
+                case_name: xrally_kubernetes_full
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    All in one tasks for checking basic functionality of
+                    Kubernetes cluster.
+                dependencies:
+                    - DEPLOY_SCENARIO: 'k8-*'
+                run:
+                    name: xrally_kubernetes
+                    args:
+                        times: 10
+                        concurrency: 4
+                        namespaces_count: 3
diff --git a/functest_kubernetes/rally/all-in-one.yaml b/functest_kubernetes/rally/all-in-one.yaml
index 9437f760..ae4157ff 100644
--- a/functest_kubernetes/rally/all-in-one.yaml
+++ b/functest_kubernetes/rally/all-in-one.yaml
@@ -8,16 +8,16 @@ subtasks:
       Kubernetes.list_namespaces: {}
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
 
   - title: Run a single workload with create/read/delete namespace
     scenario:
       Kubernetes.create_and_delete_namespace: {}
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
 
   - title: Run a single workload with create/read/delete pod
     scenario:
@@ -25,11 +25,11 @@ subtasks:
         image: kubernetes/pause
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/read/delete replication controller
@@ -39,11 +39,11 @@ subtasks:
         replicas: 2
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/scale/delete replication controller
@@ -54,11 +54,11 @@ subtasks:
         scale_replicas: 3
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/read/delete replicaset
@@ -68,11 +68,11 @@ subtasks:
         replicas: 1
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/scale/delete replicaset
@@ -83,11 +83,11 @@ subtasks:
         scale_replicas: 2
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title:
@@ -98,11 +98,11 @@ subtasks:
         mount_path: /opt/check
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: >-
@@ -125,7 +125,7 @@ subtasks:
         times: 1
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/read/delete pod with secret volume
@@ -138,11 +138,11 @@ subtasks:
         mount_path: /opt/check
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/check/delete pod with secret volume
@@ -159,11 +159,11 @@ subtasks:
         error_regexp: No such file
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: >-
@@ -188,7 +188,7 @@ subtasks:
         times: 1
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title:
@@ -206,11 +206,11 @@ subtasks:
             test
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: >-
@@ -237,7 +237,7 @@ subtasks:
         times: 1
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/read/delete deployment
@@ -247,11 +247,11 @@ subtasks:
         replicas: 2
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/rollout/delete deployment
@@ -271,11 +271,11 @@ subtasks:
               value: "true"
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/read/delete statefulset
@@ -285,11 +285,11 @@ subtasks:
         replicas: 2
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/scale/delete statefulset
@@ -300,11 +300,11 @@ subtasks:
         scale_replicas: 2
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/read/delete job
@@ -316,11 +316,11 @@ subtasks:
           - "SUCCESS"
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/check/delete clusterIP service
@@ -331,11 +331,11 @@ subtasks:
         protocol: TCP
     runner:
       constant:
-        concurrency: 1
-        times: 2
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: >-
@@ -353,7 +353,7 @@ subtasks:
         times: 1
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
 
   - title: Run a single workload with create/check/delete NodePort service
@@ -365,9 +365,9 @@ subtasks:
         request_timeout: 10
     runner:
       constant:
-        concurrency: 1
-        times: 1
+        concurrency: {{ concurrency }}
+        times: {{ times }}
     contexts:
       namespaces:
-        count: 3
+        count: {{ namespaces_count }}
         with_serviceaccount: true
diff --git a/functest_kubernetes/rally/rally_kubernetes.py b/functest_kubernetes/rally/rally_kubernetes.py
index e7a903d9..28415579 100644
--- a/functest_kubernetes/rally/rally_kubernetes.py
+++ b/functest_kubernetes/rally/rally_kubernetes.py
@@ -11,6 +11,7 @@ import logging
 import os
 import time
 
+from jinja2 import Template
 import pkg_resources
 from rally import api
 from rally import exceptions
@@ -26,6 +27,10 @@ class RallyKubernetes(testcase.TestCase):
 
     __logger = logging.getLogger(__name__)
 
+    concurrency = 1
+    times = 1
+    namespaces_count = 1
+
     def __init__(self, **kwargs):
         super(RallyKubernetes, self).__init__(**kwargs)
         self.dir_results = "/home/opnfv/functest/results"
@@ -53,10 +58,14 @@ class RallyKubernetes(testcase.TestCase):
                 "Cannot check env heath: %s",
                 result['existing@kubernetes']['message'])
             return
-        input_task = open(
-            pkg_resources.resource_filename(
-                'functest_kubernetes', 'rally/all-in-one.yaml')).read()
-        task = yaml.safe_load(input_task)
+        with open(pkg_resources.resource_filename(
+                'functest_kubernetes', 'rally/all-in-one.yaml')) as file:
+            template = Template(file.read())
+        task = yaml.safe_load(template.render(
+            concurrency=kwargs.get("concurrency", self.concurrency),
+            times=kwargs.get("times", self.times),
+            namespaces_count=kwargs.get(
+                "namespaces_count", self.namespaces_count)))
         rapi.task.validate(deployment='my-kubernetes', config=task)
         task_instance = rapi.task.create(deployment='my-kubernetes')
         rapi.task.start(
diff --git a/requirements.txt b/requirements.txt
index 742dcbf3..60c38eb2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ xtesting # Apache-2.0
 rally
 xrally-kubernetes
 kubernetes # Apache-2.0
+Jinja2 # BSD License (3 clause)
diff --git a/tox.ini b/tox.ini
index 8368ad66..602ee316 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,7 +32,6 @@ basepython = python3.6
 files =
   .travis.yml
   docker
-  functest_kubernetes/rally/all-in-one.yaml
 commands =
   yamllint {[testenv:yamllint]files}