aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorCédric Ollivier <cedric.ollivier@orange.com>2020-11-21 13:06:25 +0100
committerCédric Ollivier <cedric.ollivier@orange.com>2020-11-21 13:32:10 +0100
commitb8e9d2d899501765a5f2fca4efa2676bb0e1eb70 (patch)
treeaf896493c75d327d825ab5419adec33ce0f4d6ae
parent4a0c2be6b10107756fc4cd7e63a4beeb95462b17 (diff)
Update cnf_conformance to v0.9.4
It removes all hacks bypassing a few former bugs (e.g. [1]) and adapts to the new cnf-conformance result output logic. It runs all workload tests as it's now proposed (k8s conformance is already part of smoke) and select the latest coredns cnf-conformance.yml taking latest helm repo changes into account. It sets criteria 95 as all steps are considered as passed and a overall result 100/105. [1] https://github.com/cncf/cnf-conformance/issues/388 Change-Id: I95b7889b21b86961e7a1d0c84b13e0ae367cae44 Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com> (cherry picked from commit 854f164998793000fcf8019214b259b7283f5194)
-rw-r--r--docker/cnf/Dockerfile22
-rw-r--r--docker/cnf/testcases.yaml3
-rw-r--r--functest_kubernetes/cnf_conformance/conformance.py44
3 files changed, 32 insertions, 37 deletions
diff --git a/docker/cnf/Dockerfile b/docker/cnf/Dockerfile
index b0461ad4..b6c078a7 100644
--- a/docker/cnf/Dockerfile
+++ b/docker/cnf/Dockerfile
@@ -1,7 +1,7 @@
FROM opnfv/functest-kubernetes-core:hunter
ARG K8S_TAG=v1.13.12
-ARG CNF_CONFORMANCE_TAG=v0.7.2-beta1
+ARG CNF_CONFORMANCE_TAG=v0.9.4
ARG HELM_TAG=v3.3.1
RUN apk --no-cache add --update wget curl libc6-compat && \
@@ -20,14 +20,16 @@ RUN apk --no-cache add --update wget curl libc6-compat && \
chmod +x /usr/local/bin/kubectl /usr/local/bin/helm && \
rm -r /src/k8s.io /src/helm-$HELM_TAG-linux-$ARCH.tar.gz /src/linux-$ARCH && \
case $(uname -m) in x86_64) \
- curl https://github.com/cncf/cnf-conformance/releases/download/$CNF_CONFORMANCE_TAG/cnf-conformance-$(echo $CNF_CONFORMANCE_TAG |cut -d\- -f 1)-$ARCH-static \
- -Ls --output /usr/local/bin/cnf-conformance && \
- chmod +x /usr/local/bin/cnf-conformance ;; esac && \
- mkdir -p /src/cnf-conformance && \
- curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/$CNF_CONFORMANCE_TAG/example-cnfs/coredns/cnf-conformance.yml \
- -s --output /src/cnf-conformance/cnf-conformance.yml && \
- curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/$CNF_CONFORMANCE_TAG/points.yml \
- -s --output /src/cnf-conformance/points.yml && \
- apk del .build-deps
+ curl https://github.com/cncf/cnf-conformance/releases/download/$CNF_CONFORMANCE_TAG/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz \
+ -Ls --output /src/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz && \
+ tar zxf /src/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz ./cnf-conformance -C /usr/local/bin && \
+ chmod +x /usr/local/bin/cnf-conformance && \
+ mkdir /src/cnf-conformance && \
+ curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/e636baff623dd934eadfce77891fc54da08a8134/example-cnfs/coredns/cnf-conformance.yml \
+ --output /src/cnf-conformance/cnf-conformance.yml && \
+ curl -Ls https://raw.githubusercontent.com/cncf/cnf-conformance/$CNF_CONFORMANCE_TAG/points.yml \
+ --output /src/cnf-conformance/points.yml && \
+ helm repo add stable https://cncf.gitlab.io/stable && \
+ rm /src/cnf-conformance-$CNF_CONFORMANCE_TAG.tar.gz ;; esac
COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
CMD ["run_tests", "-t", "all"]
diff --git a/docker/cnf/testcases.yaml b/docker/cnf/testcases.yaml
index f04eb6f1..62ae54bc 100644
--- a/docker/cnf/testcases.yaml
+++ b/docker/cnf/testcases.yaml
@@ -34,8 +34,7 @@ tiers:
-
case_name: cnf_conformance
project_name: functest
- enabled: false
- criteria: 75
+ criteria: 95
blocking: false
description: >-
Enable interoperability of Cloud native Network
diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py
index f342fcc6..564f279d 100644
--- a/functest_kubernetes/cnf_conformance/conformance.py
+++ b/functest_kubernetes/cnf_conformance/conformance.py
@@ -16,7 +16,7 @@ different vendors [1].
from __future__ import division
-import fnmatch
+import glob
import logging
import os
import re
@@ -38,7 +38,7 @@ class CNFConformance(testcase.TestCase):
src_dir = '/src/cnf-conformance'
bin_dir = '/usr/local/bin'
- default_tag = 'all'
+ default_tag = 'workload'
__logger = logging.getLogger(__name__)
@@ -64,12 +64,6 @@ class CNFConformance(testcase.TestCase):
shutil.copy2(
os.path.join(self.src_dir, 'cnf-conformance.yml'), self.res_dir)
os.chdir(self.res_dir)
- # cnf-conformance must be in the working dir
- # https://github.com/cncf/cnf-conformance/issues/388
- if not os.path.exists(os.path.join(self.res_dir, 'cnf-conformance')):
- os.symlink(
- os.path.join(self.bin_dir, 'cnf-conformance'),
- os.path.join(self.res_dir, 'cnf-conformance'))
cmd = ['cnf-conformance', 'setup']
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
@@ -80,25 +74,26 @@ class CNFConformance(testcase.TestCase):
def run_conformance(self, **kwargs):
"""Run CNF Conformance"""
- # a previous results.yml leads to interactive mode
- if os.path.exists(os.path.join(self.res_dir, 'results.yml')):
- os.remove(os.path.join(self.res_dir, 'results.yml'))
cmd = ['cnf-conformance', kwargs.get("tag", self.default_tag)]
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
- for lfile in os.listdir(self.res_dir):
- if fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
- with open(os.path.join(self.res_dir, lfile)) as yfile:
- self.details = yaml.safe_load(yfile)
- msg = prettytable.PrettyTable(
- header_style='upper', padding_width=5,
- field_names=['name', 'status'])
- for item in self.details['items']:
- msg.add_row([item['name'], item['status']])
- self.__logger.info("\n\n%s\n", msg.get_string())
- grp = re.search(r'Final score: (\d+) of (\d+)', output.decode("utf-8"))
+ lfiles = glob.glob(os.path.join(
+ self.res_dir, 'results', 'cnf-conformance-results-*.yml'))
+ results = max(lfiles, key=os.path.getmtime)
+ with open(os.path.join(self.res_dir, 'results', results)) as yfile:
+ self.details = yaml.safe_load(yfile)
+ msg = prettytable.PrettyTable(
+ header_style='upper', padding_width=5,
+ field_names=['name', 'status'])
+ for item in self.details['items']:
+ msg.add_row([item['name'], item['status']])
+ self.__logger.info("\n\n%s\n", msg.get_string())
+ grp = re.search(
+ r'Final .* score: (\d+) of (\d+)', output.decode("utf-8"))
if grp:
self.result = int(grp.group(1)) / int(grp.group(2)) * 100
+ shutil.copy2(
+ os.path.join(self.res_dir, 'results', results), self.res_dir)
def run(self, **kwargs):
""""Running the test with example CNF"""
@@ -117,6 +112,5 @@ class CNFConformance(testcase.TestCase):
self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
shutil.rmtree(os.path.join(self.res_dir, 'tools'), ignore_errors=True)
shutil.rmtree(os.path.join(self.res_dir, 'cnfs'), ignore_errors=True)
- for lfile in os.listdir(self.res_dir):
- if not fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
- os.remove(os.path.join(self.res_dir, lfile))
+ shutil.rmtree(
+ os.path.join(self.res_dir, 'results'), ignore_errors=True)