aboutsummaryrefslogtreecommitdiffstats
path: root/functest_kubernetes/cnf_conformance
diff options
context:
space:
mode:
authorCédric Ollivier <cedric.ollivier@orange.com>2020-11-21 13:06:25 +0100
committerCédric Ollivier <cedric.ollivier@orange.com>2020-11-21 13:28:15 +0100
commit854f164998793000fcf8019214b259b7283f5194 (patch)
treecb6207835d3d64f3c7e99e8b09f20d5933a7c236 /functest_kubernetes/cnf_conformance
parent30a5a084fa9e0c8677d70d8b1d7a696ea06b49b9 (diff)
Update cnf_conformance to v0.9.4
It removes all hacks bypassing a few former bugs (e.g. [1]) and adapts to the new cnf-conformance result output logic. It runs all workload tests as it's now proposed (k8s conformance is already part of smoke) and select the latest coredns cnf-conformance.yml taking latest helm repo changes into account. It sets criteria 95 as all steps are considered as passed and a overall result 100/105. [1] https://github.com/cncf/cnf-conformance/issues/388 Change-Id: I95b7889b21b86961e7a1d0c84b13e0ae367cae44 Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
Diffstat (limited to 'functest_kubernetes/cnf_conformance')
-rw-r--r--functest_kubernetes/cnf_conformance/conformance.py44
1 files changed, 19 insertions, 25 deletions
diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py
index f342fcc6..564f279d 100644
--- a/functest_kubernetes/cnf_conformance/conformance.py
+++ b/functest_kubernetes/cnf_conformance/conformance.py
@@ -16,7 +16,7 @@ different vendors [1].
from __future__ import division
-import fnmatch
+import glob
import logging
import os
import re
@@ -38,7 +38,7 @@ class CNFConformance(testcase.TestCase):
src_dir = '/src/cnf-conformance'
bin_dir = '/usr/local/bin'
- default_tag = 'all'
+ default_tag = 'workload'
__logger = logging.getLogger(__name__)
@@ -64,12 +64,6 @@ class CNFConformance(testcase.TestCase):
shutil.copy2(
os.path.join(self.src_dir, 'cnf-conformance.yml'), self.res_dir)
os.chdir(self.res_dir)
- # cnf-conformance must be in the working dir
- # https://github.com/cncf/cnf-conformance/issues/388
- if not os.path.exists(os.path.join(self.res_dir, 'cnf-conformance')):
- os.symlink(
- os.path.join(self.bin_dir, 'cnf-conformance'),
- os.path.join(self.res_dir, 'cnf-conformance'))
cmd = ['cnf-conformance', 'setup']
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
@@ -80,25 +74,26 @@ class CNFConformance(testcase.TestCase):
def run_conformance(self, **kwargs):
"""Run CNF Conformance"""
- # a previous results.yml leads to interactive mode
- if os.path.exists(os.path.join(self.res_dir, 'results.yml')):
- os.remove(os.path.join(self.res_dir, 'results.yml'))
cmd = ['cnf-conformance', kwargs.get("tag", self.default_tag)]
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
- for lfile in os.listdir(self.res_dir):
- if fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
- with open(os.path.join(self.res_dir, lfile)) as yfile:
- self.details = yaml.safe_load(yfile)
- msg = prettytable.PrettyTable(
- header_style='upper', padding_width=5,
- field_names=['name', 'status'])
- for item in self.details['items']:
- msg.add_row([item['name'], item['status']])
- self.__logger.info("\n\n%s\n", msg.get_string())
- grp = re.search(r'Final score: (\d+) of (\d+)', output.decode("utf-8"))
+ lfiles = glob.glob(os.path.join(
+ self.res_dir, 'results', 'cnf-conformance-results-*.yml'))
+ results = max(lfiles, key=os.path.getmtime)
+ with open(os.path.join(self.res_dir, 'results', results)) as yfile:
+ self.details = yaml.safe_load(yfile)
+ msg = prettytable.PrettyTable(
+ header_style='upper', padding_width=5,
+ field_names=['name', 'status'])
+ for item in self.details['items']:
+ msg.add_row([item['name'], item['status']])
+ self.__logger.info("\n\n%s\n", msg.get_string())
+ grp = re.search(
+ r'Final .* score: (\d+) of (\d+)', output.decode("utf-8"))
if grp:
self.result = int(grp.group(1)) / int(grp.group(2)) * 100
+ shutil.copy2(
+ os.path.join(self.res_dir, 'results', results), self.res_dir)
def run(self, **kwargs):
""""Running the test with example CNF"""
@@ -117,6 +112,5 @@ class CNFConformance(testcase.TestCase):
self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
shutil.rmtree(os.path.join(self.res_dir, 'tools'), ignore_errors=True)
shutil.rmtree(os.path.join(self.res_dir, 'cnfs'), ignore_errors=True)
- for lfile in os.listdir(self.res_dir):
- if not fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
- os.remove(os.path.join(self.res_dir, lfile))
+ shutil.rmtree(
+ os.path.join(self.res_dir, 'results'), ignore_errors=True)