Add Kubernetes conformance tests

JIRA: FUNCTEST-905

Change-Id: I2f70884811431f21fe98bb6a989a9172f3612a64
Signed-off-by: djkonro <konraddjimeli@gmail.com>

4 files changed, 98 insertions, 15 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 9aa7f84e..d860bb93 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -9,7 +9,8 @@ RUN apk --no-cache add --update make bash go \
     (cd /src/k8s.io/kubernetes && git checkout $K8S_TAG) && \
     (cd /src/k8s.io/kubernetes && \
         make kubectl ginkgo && \
-        make WHAT=test/e2e/e2e.test) && \
+        make WHAT=test/e2e/e2e.test && \
+        mv _output/bin/kubectl /usr/local/bin) && \
     git clone https://gerrit.opnfv.org/gerrit/functest-kubernetes /src/functest-kubernetes && \
     (cd /src/functest-kubernetes && git fetch origin $BRANCH && git checkout FETCH_HEAD) && \
     pip install /src/functest-kubernetes && \
diff --git a/docker/logging.ini b/docker/logging.ini
index 594de7af..36e0bb49 100644
--- a/docker/logging.ini
+++ b/docker/logging.ini
@@ -1,8 +1,8 @@
 [loggers]
-keys=root,functest,functest_kubernetes
+keys=root,functest,functest_kubernetes,k8s_smoke,k8s_conformance
 
 [handlers]
-keys=console,wconsole,file,null
+keys=console,wconsole,file,null,k8s_smoke_logfile,k8s_conformance_logfile
 
 [formatters]
 keys=standard
@@ -13,7 +13,7 @@ handlers=null
 
 [logger_functest_kubernetes]
 level=NOTSET
-handlers=console
+handlers=console,file
 qualname=functest_kubernetes
 
 [logger_functest]
@@ -21,6 +21,16 @@ level=NOTSET
 handlers=console
 qualname=functest
 
+[logger_k8s_smoke]
+level=NOTSET
+handlers=k8s_smoke_logfile
+qualname=k8s_smoke
+
+[logger_k8s_conformance]
+level=NOTSET
+handlers=k8s_conformance_logfile
+qualname=k8s_conformance
+
 [handler_null]
 class=NullHandler
 level=NOTSET
@@ -45,6 +55,18 @@ level=DEBUG
 formatter=standard
 args=("/home/opnfv/functest/results/functest-kubernetes.log",)
 
+[handler_k8s_smoke_logfile]
+class=FileHandler
+level=DEBUG
+formatter=standard
+args=("/home/opnfv/functest/results/k8s_smoke.log",)
+
+[handler_k8s_conformance_logfile]
+class=FileHandler
+level=DEBUG
+formatter=standard
+args=("/home/opnfv/functest/results/k8s_conformance.log",)
+
 [formatter_standard]
 format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
 datefmt=
diff --git a/docker/testcases.yaml b/docker/testcases.yaml
index 2eed64c1..493eb8b7 100644
--- a/docker/testcases.yaml
+++ b/docker/testcases.yaml
@@ -22,3 +22,19 @@ tiers:
                 run:
                     module: 'functest_kubernetes.k8stest'
                     class: 'K8sSmokeTest'
+
+            -
+                case_name: k8s_conformance
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    Conformance Tests are expect to pass on any Kubernetes
+                    cluster, this tests are necessary demonstrate conformance
+                    of the Kubernetes deployment.
+                dependencies:
+                    installer: '(compass)|(joid)'
+                    scenario: 'k8-*'
+                run:
+                    module: 'functest_kubernetes.k8stest'
+                    class: 'K8sConformanceTest'
diff --git a/functest_kubernetes/k8stest.py b/functest_kubernetes/k8stest.py
index ce343a20..8fe8d5cd 100644
--- a/functest_kubernetes/k8stest.py
+++ b/functest_kubernetes/k8stest.py
@@ -16,6 +16,7 @@ from __future__ import division
 
 import logging
 import os
+import re
 import subprocess
 import time
 
@@ -35,26 +36,58 @@ class K8sTesting(testcase.TestCase):
         self.start_time = 0
         self.stop_time = 0
 
-    def run_kubetest(self):
+    def run_kubetest(self):  # pylint: disable=too-many-branches
         """Run the test suites"""
         cmd_line = self.cmd
         LOGGER.info("Starting k8s test: '%s'.", cmd_line)
 
         process = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)
-        remark = []
-        lines = process.stdout.readlines()
-        for i in range(len(lines) - 1, -1, -1):
-            new_line = str(lines[i])
-
-            if 'SUCCESS!' in new_line or 'FAIL!' in new_line:
-                remark = new_line.replace('--', '|').split('|')
+        output = process.stdout.read()
+        # Remove color code escape sequences
+        output = re.sub(r'\x1B\[[0-?]*[ -/]*[@-~]', '', str(output))
+
+        file_logger = logging.getLogger(self.case_name)
+        remarks = []
+        lines = output.split('\n')
+        i = 0
+        while i < len(lines):
+            if '[k8s.io]' in lines[i]:
+                if i != 0 and 'seconds' in lines[i-1]:
+                    file_logger.debug(lines[i-1])
+                while lines[i] != '-'*len(lines[i]):
+                    if lines[i].startswith('STEP:') or ('INFO:' in lines[i]):
+                        break
+                    file_logger.debug(lines[i])
+                    i = i+1
+
+            success = 'SUCCESS!' in lines[i]
+            failure = 'FAIL!' in lines[i]
+            if success or failure:
+                if i != 0 and 'seconds' in lines[i-1]:
+                    remarks.append(lines[i-1])
+                remarks = remarks + lines[i].replace('--', '|').split('|')
                 break
-
-        if remark and 'SUCCESS!' in remark[0]:
+            i = i+1
+
+        file_logger.debug('-'*10)
+        file_logger.debug("Remarks:")
+        for remark in remarks:
+            if 'seconds' in remark:
+                file_logger.debug(remark)
+            elif 'Passed' in remark:
+                file_logger.debug("Passed: %s", remark.split()[0])
+            elif 'Skipped' in remark:
+                file_logger.debug("Skipped: %s", remark.split()[0])
+            elif 'Failed' in remark:
+                file_logger.debug("Failed: %s", remark.split()[0])
+
+        if success:
             self.result = 100
+        elif failure:
+            self.result = 0
 
-    def run(self):
+    def run(self, **kwargs):
 
         if not os.path.isfile(os.getenv('KUBECONFIG')):
             LOGGER.error("Cannot run k8s testcases. Config file not found ")
@@ -93,3 +126,14 @@ class K8sSmokeTest(K8sTesting):
         self.check_envs()
         self.cmd = ['/src/k8s.io/kubernetes/cluster/test-smoke.sh', '--host',
                     os.getenv('KUBE_MASTER_URL')]
+
+
+class K8sConformanceTest(K8sTesting):
+    """Kubernetes conformance test suite"""
+    def __init__(self, **kwargs):
+        if "case_name" not in kwargs:
+            kwargs.get("case_name", 'k8s_conformance')
+        super(K8sConformanceTest, self).__init__(**kwargs)
+        self.check_envs()
+        self.cmd = ['/src/k8s.io/kubernetes/_output/bin/e2e.test',
+                    '--ginkgo.focus', 'Conformance']