Update cnf_conformance to v0.9.4

It removes all hacks bypassing a few former bugs (e.g. [1]) and adapts
to the new cnf-conformance result output logic.

It runs all workload tests as it's now proposed (k8s conformance
is already part of smoke) and select the latest coredns
cnf-conformance.yml taking latest helm repo changes into account.

It sets criteria 95 as all steps are considered as passed and a
overall result 100/105.

[1] https://github.com/cncf/cnf-conformance/issues/388

Change-Id: I95b7889b21b86961e7a1d0c84b13e0ae367cae44
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

1 files changed, 19 insertions, 25 deletions

diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py
index f342fcc6..564f279d 100644
--- a/functest_kubernetes/cnf_conformance/conformance.py
+++ b/functest_kubernetes/cnf_conformance/conformance.py
@@ -16,7 +16,7 @@ different vendors [1].
 
 from __future__ import division
 
-import fnmatch
+import glob
 import logging
 import os
 import re
@@ -38,7 +38,7 @@ class CNFConformance(testcase.TestCase):
 
     src_dir = '/src/cnf-conformance'
     bin_dir = '/usr/local/bin'
-    default_tag = 'all'
+    default_tag = 'workload'
 
     __logger = logging.getLogger(__name__)
 
@@ -64,12 +64,6 @@ class CNFConformance(testcase.TestCase):
         shutil.copy2(
             os.path.join(self.src_dir, 'cnf-conformance.yml'), self.res_dir)
         os.chdir(self.res_dir)
-        # cnf-conformance must be in the working dir
-        # https://github.com/cncf/cnf-conformance/issues/388
-        if not os.path.exists(os.path.join(self.res_dir, 'cnf-conformance')):
-            os.symlink(
-                os.path.join(self.bin_dir, 'cnf-conformance'),
-                os.path.join(self.res_dir, 'cnf-conformance'))
         cmd = ['cnf-conformance', 'setup']
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
@@ -80,25 +74,26 @@ class CNFConformance(testcase.TestCase):
 
     def run_conformance(self, **kwargs):
         """Run CNF Conformance"""
-        # a previous results.yml leads to interactive mode
-        if os.path.exists(os.path.join(self.res_dir, 'results.yml')):
-            os.remove(os.path.join(self.res_dir, 'results.yml'))
         cmd = ['cnf-conformance', kwargs.get("tag", self.default_tag)]
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
-        for lfile in os.listdir(self.res_dir):
-            if fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
-                with open(os.path.join(self.res_dir, lfile)) as yfile:
-                    self.details = yaml.safe_load(yfile)
-                    msg = prettytable.PrettyTable(
-                        header_style='upper', padding_width=5,
-                        field_names=['name', 'status'])
-                    for item in self.details['items']:
-                        msg.add_row([item['name'], item['status']])
-                    self.__logger.info("\n\n%s\n", msg.get_string())
-        grp = re.search(r'Final score: (\d+) of (\d+)', output.decode("utf-8"))
+        lfiles = glob.glob(os.path.join(
+            self.res_dir, 'results', 'cnf-conformance-results-*.yml'))
+        results = max(lfiles, key=os.path.getmtime)
+        with open(os.path.join(self.res_dir, 'results', results)) as yfile:
+            self.details = yaml.safe_load(yfile)
+            msg = prettytable.PrettyTable(
+                header_style='upper', padding_width=5,
+                field_names=['name', 'status'])
+            for item in self.details['items']:
+                msg.add_row([item['name'], item['status']])
+            self.__logger.info("\n\n%s\n", msg.get_string())
+        grp = re.search(
+            r'Final .* score: (\d+) of (\d+)', output.decode("utf-8"))
         if grp:
             self.result = int(grp.group(1)) / int(grp.group(2)) * 100
+        shutil.copy2(
+            os.path.join(self.res_dir, 'results', results), self.res_dir)
 
     def run(self, **kwargs):
         """"Running the test with example CNF"""
@@ -117,6 +112,5 @@ class CNFConformance(testcase.TestCase):
         self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
         shutil.rmtree(os.path.join(self.res_dir, 'tools'), ignore_errors=True)
         shutil.rmtree(os.path.join(self.res_dir, 'cnfs'), ignore_errors=True)
-        for lfile in os.listdir(self.res_dir):
-            if not fnmatch.fnmatch(lfile, 'cnf-conformance-results-*.yml'):
-                os.remove(os.path.join(self.res_dir, lfile))
+        shutil.rmtree(
+            os.path.join(self.res_dir, 'results'), ignore_errors=True)