diff options
Diffstat (limited to 'functest_kubernetes/cnf_conformance/conformance.py')
| -rw-r--r-- | functest_kubernetes/cnf_conformance/conformance.py | 59 |
1 files changed, 27 insertions, 32 deletions
diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py index d1e4bcf1..289a8f5a 100644 --- a/functest_kubernetes/cnf_conformance/conformance.py +++ b/functest_kubernetes/cnf_conformance/conformance.py @@ -41,6 +41,7 @@ class CNFConformance(testcase.TestCase): src_dir = '/src/cnf-testsuite' bin_dir = '/usr/local/bin' default_tag = 'cert' + default_cnf_config = 'example-cnfs/coredns/cnf-testsuite.yml' __logger = logging.getLogger(__name__) @@ -50,6 +51,7 @@ class CNFConformance(testcase.TestCase): self.corev1 = client.CoreV1Api() self.output_log_name = 'functest-kubernetes.log' self.output_debug_log_name = 'functest-kubernetes.debug.log' + self.cnf_config = '' def check_requirements(self): """Check if cnf-testsuite is in $PATH""" @@ -63,27 +65,6 @@ class CNFConformance(testcase.TestCase): """Implement initialization and pre-reqs steps""" if os.path.exists(os.path.join(self.src_dir, "results")): shutil.rmtree(os.path.join(self.src_dir, "results")) - api_response = self.corev1.list_namespace() - for namespace in ["cnf-testsuite", "cnf-default", "litmus"]: - for item in api_response.items: - if item.metadata.name == namespace: - self.corev1.patch_namespace( - namespace, - client.V1Namespace(metadata=client.V1ObjectMeta( - labels={ - "pod-security.kubernetes.io/enforce": - "privileged"}))) - self.__logger.debug( - "patch_namespace: %s", namespace) - break - else: - self.corev1.create_namespace( - client.V1Namespace(metadata=client.V1ObjectMeta( - name=namespace, labels={ - "pod-security.kubernetes.io/enforce": - "privileged"}))) - self.__logger.debug( - "create_namespace: %s", namespace) os.chdir(self.src_dir) cmd = ['cnf-testsuite', 'setup', '-l', 'debug'] try: @@ -95,8 +76,8 @@ class CNFConformance(testcase.TestCase): self.result = 0 return False self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8")) - cmd = ['cnf-testsuite', 'cnf_setup', - 'cnf-config=cnf-testsuite.yml', '-l', 'debug'] + cmd = ['cnf-testsuite', 'cnf_install', + f'cnf-config={self.cnf_config}', '-l', 'debug'] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: @@ -134,13 +115,21 @@ class CNFConformance(testcase.TestCase): self.__logger.warning( "%s %s", item['name'], item['status']) self.__logger.info("\n\n%s\n", msg.get_string()) - grp = re.search( - r'(\d+) of (\d+) essential tests passed', output.decode("utf-8")) - if grp: - # https://github.com/cncf/cnf-certification/blob/main/reviewing.md - self.result = int(grp.group(1)) + if kwargs.get("tag", self.default_tag) == 'cert': + grp = re.search( + r'(\d+) of (\d+) essential tests passed', + output.decode("utf-8")) + if grp: + self.result = int(grp.group(1)) + else: + self.result = 0 else: - self.result = 0 + grp = re.search( + r'Final .* score: (\d+) of (\d+)', output.decode("utf-8")) + if grp: + self.result = int(grp.group(1)) / int(grp.group(2)) * 100 + else: + self.result = item_criteria / len(self.details['items']) * 100 if not os.path.exists(self.res_dir): os.makedirs(self.res_dir) shutil.copy2( @@ -150,19 +139,25 @@ class CNFConformance(testcase.TestCase): def run(self, **kwargs): """"Running the test with example CNF""" self.start_time = time.time() + self.cnf_config = kwargs.get("cnf-config", self.default_cnf_config) if self.setup(): self.run_conformance(**kwargs) self.stop_time = time.time() def clean(self): - for clean_cmd in ['cnf_cleanup']: + for clean_cmd in ['cnf_uninstall', 'uninstall_all']: cmd = ['cnf-testsuite', clean_cmd, - 'cnf-config=cnf-testsuite.yml'] + f'cnf-config={self.cnf_config}'] output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8")) try: - for namespace in ["cnf-testsuite", "cnf-default", "litmus"]: + for namespace in ["cnf-testsuite", "cnf-default"]: self.corev1.delete_namespace(namespace) self.__logger.debug("delete_namespace: %s", namespace) except client.rest.ApiException: pass + # cnf-testsuite cleanup does not wait for removing reources + # use time.sleep(60) till + # https://github.com/cnti-testcatalog/testsuite/issues/2194 + # is fixed + time.sleep(60) |