diff options
| -rw-r--r-- | README.md | 86 | ||||
| -rw-r--r-- | docker/cnf/Dockerfile | 6 | ||||
| -rw-r--r-- | docker/cnf/testcases.yaml | 26 | ||||
| -rwxr-xr-x | elements/functest-kubernetes/install.d/16-functest-kubernetes | 5 | ||||
| -rw-r--r-- | functest_kubernetes/cnf_conformance/conformance.py | 59 | ||||
| -rw-r--r-- | tox.ini | 2 |
6 files changed, 134 insertions, 50 deletions
@@ -125,16 +125,16 @@ sudo docker run --env-file env \ ``` ``` -+-----------------------+------------------+--------------+------------------+----------------+ -| TEST CASE | PROJECT | TIER | DURATION | RESULT | -+-----------------------+------------------+--------------+------------------+----------------+ -| k8s_vims | functest | cnf | 09:06 | PASS | -| helm_vims | functest | cnf | 08:54 | PASS | -| cnf_testsuite | functest | cnf | 16:47 | PASS | -+-----------------------+------------------+--------------+------------------+----------------+ ++--------------------------------+------------------+--------------+------------------+----------------+ +| TEST CASE | PROJECT | TIER | DURATION | RESULT | ++--------------------------------+------------------+--------------+------------------+----------------+ +| k8s_vims | functest | cnf | 10:02 | PASS | +| helm_vims | functest | cnf | 07:20 | PASS | +| cnf_testsuite | functest | cnf | 06:31 | PASS | +| cnf_testsuite_workload | functest | cnf | 16:01 | PASS | ++--------------------------------+------------------+--------------+------------------+----------------+ ``` - ## Use on air gap environments (no access to Internet) To test a Kubernetes without access to Internet, repository mirrors needs to be @@ -160,3 +160,73 @@ a mirror on docker. An example on how to set it up on docker daemon is provided here: [daemon-configuration-file]( https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file) + + +## Check how cloud native your CNFs are + +Functest Kubernetes integrates the CNTi Test Catalog which is an open source +and vendor neutral tool that can be used to validate a telco application's +adherence to cloud native principles and best practices. + +Both cnf_testsuite and cnf_testsuite_workload leverages CoreDNS as the target +CNF but you're free to validate your own CNFs as the following example. + +Download cnf-testsuite.yml and the helm dir to test Envoy +```bash +git clone https://github.com/cnti-testcatalog/testsuite.git +mv testsuite/example-cnfs/envoy . +rm -rf testsuite +``` + +Write testcases.yaml +```bash +cat << EOF > testcases.yaml +--- +tiers: + - name: cnf + testcases: + - case_name: cnf_testsuite + project_name: functest + blocking: false + criteria: 15 + run: + name: cnf_testsuite + args: + cnf-config: /src/envoy/cnf-testsuite.yml + tag: cert + - case_name: cnf_testsuite_workload + project_name: functest + blocking: false + criteria: 50 + run: + name: cnf_testsuite + args: + cnf-config: /src/envoy/cnf-testsuite.yml + tag: workload +EOF +``` + +Run the CNTi certification and the CNTi workload suite +```bash +sudo docker run \ + -v $(pwd)/config:/home/xtesting/.kube/config \ + -v $(pwd)/envoy:/src/envoy \ + -v $(pwd)/testcases.yaml:/etc/xtesting/testcases.yaml \ + opnfv/functest-kubernetes-cnf +``` + +``` ++--------------------------------+------------------+--------------+------------------+----------------+ +| TEST CASE | PROJECT | TIER | DURATION | RESULT | ++--------------------------------+------------------+--------------+------------------+----------------+ +| cnf_testsuite | functest | cnf | 08:11 | FAIL | +| cnf_testsuite_workload | functest | cnf | 16:14 | PASS | ++--------------------------------+------------------+--------------+------------------+----------------+ +``` + +Please note that Envoy as proposed as example fails the +[CNTi certification](https://github.com/lfn-cnti/certification/blob/main/docs/CNTiCertification-2.0-beta.md) +which requires passing at least 15 of the 19 total Essential tests. +It scores 70% on workload gouping the following +[test categories](https://github.com/cnti-testcatalog/testsuite/blob/main/docs/TEST_DOCUMENTATION.md): +compatibility, state, security, configuration, observability, microservice and resilience. diff --git a/docker/cnf/Dockerfile b/docker/cnf/Dockerfile index 3e60ca64..deb02d2e 100644 --- a/docker/cnf/Dockerfile +++ b/docker/cnf/Dockerfile @@ -1,7 +1,7 @@ FROM opnfv/functest-kubernetes-core:v1.29 ARG K8S_TAG=1.29 -ARG CNF_TESTSUITE_TAG=v1.3.3 +ARG CNF_TESTSUITE_TAG=v1.4.1 ARG HELM_TAG=v3.6.0 USER root @@ -27,11 +27,7 @@ RUN apk --no-cache add --update wget curl libc6-compat ncurses && \ git fetch --tags https://github.com/cnti-testcatalog/testsuite.git $CNF_TESTSUITE_TAG && \ git checkout FETCH_HEAD) && \ chown -R xtesting: /src/cnf-testsuite && \ - ln -s /src/cnf-testsuite/example-cnfs/coredns/cnf-testsuite.yml /src/cnf-testsuite/cnf-testsuite.yml && \ - ln -s /src/cnf-testsuite/points-all.yml /src/cnf-testsuite/points.yml && \ - ln -s /usr/local/bin/cnf-testsuite /src/cnf-testsuite/cnf-testsuite && \ rm -rf /src/cnf-testsuite-$CNF_TESTSUITE_TAG.tar.gz /src/cnf-testsuite/.git ;; esac USER xtesting -RUN helm repo add stable https://cncf.gitlab.io/stable COPY testcases.yaml /etc/xtesting/testcases.yaml CMD ["run_tests", "-t", "all"] diff --git a/docker/cnf/testcases.yaml b/docker/cnf/testcases.yaml index 30b6b664..704f2e9d 100644 --- a/docker/cnf/testcases.yaml +++ b/docker/cnf/testcases.yaml @@ -30,14 +30,34 @@ tiers: - case_name: cnf_testsuite project_name: functest - # https://github.com/cncf/cnf-certification/blob/main/reviewing.md - criteria: 10 + # https://github.com/lfn-cnti/certification/blob/main/docs/CNTiCertification-2.0-beta.md + # V2.0-beta certification requires passing at least 15 of + # the 19 total Essential tests. + criteria: 15 blocking: false description: >- Enable interoperability of Cloud native Network Functions (CNFs) as proposed by - https://github.com/cncf/cnf-testsuite + https://github.com/cnti-testcatalog/testsuite dependencies: - DEPLOY_SCENARIO: "k8-*" run: name: cnf_testsuite + args: + cnf-config: example-cnfs/coredns/cnf-testsuite.yml + tag: cert + - case_name: cnf_testsuite_workload + project_name: functest + criteria: 50 + blocking: false + description: >- + Enable interoperability of Cloud native Network + Functions (CNFs) as proposed by + https://github.com/cnti-testcatalog/testsuite + dependencies: + - DEPLOY_SCENARIO: "k8-*" + run: + name: cnf_testsuite + args: + cnf-config: example-cnfs/coredns/cnf-testsuite.yml + tag: workload diff --git a/elements/functest-kubernetes/install.d/16-functest-kubernetes b/elements/functest-kubernetes/install.d/16-functest-kubernetes index 02200b15..43a64b1d 100755 --- a/elements/functest-kubernetes/install.d/16-functest-kubernetes +++ b/elements/functest-kubernetes/install.d/16-functest-kubernetes @@ -6,7 +6,10 @@ download-frozen-image-v2.sh /data/docker/functest-kubernetes \ opnfv/functest-kubernetes-benchmarking:v1.29 \ opnfv/functest-kubernetes-security:v1.29 \ opnfv/functest-kubernetes-cnf:v1.29 -git clone https://git.opnfv.org/functest-kubernetes /home/debian/functest-kubernetes +git init /home/debian/functest-kubernetes && \ +(cd /home/debian/functest-kubernetes && \ + git fetch --tags https://git.opnfv.org/functest-kubernetes stable/v1.29 && \ + git checkout FETCH_HEAD) && \ chown -R 1000:1000 /home/debian/functest-kubernetes exit 0 diff --git a/functest_kubernetes/cnf_conformance/conformance.py b/functest_kubernetes/cnf_conformance/conformance.py index 41ce05c8..289a8f5a 100644 --- a/functest_kubernetes/cnf_conformance/conformance.py +++ b/functest_kubernetes/cnf_conformance/conformance.py @@ -41,6 +41,7 @@ class CNFConformance(testcase.TestCase): src_dir = '/src/cnf-testsuite' bin_dir = '/usr/local/bin' default_tag = 'cert' + default_cnf_config = 'example-cnfs/coredns/cnf-testsuite.yml' __logger = logging.getLogger(__name__) @@ -50,6 +51,7 @@ class CNFConformance(testcase.TestCase): self.corev1 = client.CoreV1Api() self.output_log_name = 'functest-kubernetes.log' self.output_debug_log_name = 'functest-kubernetes.debug.log' + self.cnf_config = '' def check_requirements(self): """Check if cnf-testsuite is in $PATH""" @@ -63,27 +65,6 @@ class CNFConformance(testcase.TestCase): """Implement initialization and pre-reqs steps""" if os.path.exists(os.path.join(self.src_dir, "results")): shutil.rmtree(os.path.join(self.src_dir, "results")) - api_response = self.corev1.list_namespace() - for namespace in ["cnf-testsuite", "default", "litmus"]: - for item in api_response.items: - if item.metadata.name == namespace: - self.corev1.patch_namespace( - namespace, - client.V1Namespace(metadata=client.V1ObjectMeta( - labels={ - "pod-security.kubernetes.io/enforce": - "privileged"}))) - self.__logger.debug( - "patch_namespace: %s", namespace) - break - else: - self.corev1.create_namespace( - client.V1Namespace(metadata=client.V1ObjectMeta( - name=namespace, labels={ - "pod-security.kubernetes.io/enforce": - "privileged"}))) - self.__logger.debug( - "create_namespace: %s", namespace) os.chdir(self.src_dir) cmd = ['cnf-testsuite', 'setup', '-l', 'debug'] try: @@ -95,8 +76,8 @@ class CNFConformance(testcase.TestCase): self.result = 0 return False self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8")) - cmd = ['cnf-testsuite', 'cnf_setup', - 'cnf-config=cnf-testsuite.yml', '-l', 'debug'] + cmd = ['cnf-testsuite', 'cnf_install', + f'cnf-config={self.cnf_config}', '-l', 'debug'] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: @@ -134,13 +115,21 @@ class CNFConformance(testcase.TestCase): self.__logger.warning( "%s %s", item['name'], item['status']) self.__logger.info("\n\n%s\n", msg.get_string()) - grp = re.search( - r'(\d+) of (\d+) essential tests passed', output.decode("utf-8")) - if grp: - # https://github.com/cncf/cnf-certification/blob/main/reviewing.md - self.result = int(grp.group(1)) + if kwargs.get("tag", self.default_tag) == 'cert': + grp = re.search( + r'(\d+) of (\d+) essential tests passed', + output.decode("utf-8")) + if grp: + self.result = int(grp.group(1)) + else: + self.result = 0 else: - self.result = 0 + grp = re.search( + r'Final .* score: (\d+) of (\d+)', output.decode("utf-8")) + if grp: + self.result = int(grp.group(1)) / int(grp.group(2)) * 100 + else: + self.result = item_criteria / len(self.details['items']) * 100 if not os.path.exists(self.res_dir): os.makedirs(self.res_dir) shutil.copy2( @@ -150,19 +139,25 @@ class CNFConformance(testcase.TestCase): def run(self, **kwargs): """"Running the test with example CNF""" self.start_time = time.time() + self.cnf_config = kwargs.get("cnf-config", self.default_cnf_config) if self.setup(): self.run_conformance(**kwargs) self.stop_time = time.time() def clean(self): - for clean_cmd in ['cnf_cleanup']: + for clean_cmd in ['cnf_uninstall', 'uninstall_all']: cmd = ['cnf-testsuite', clean_cmd, - 'cnf-config=cnf-testsuite.yml'] + f'cnf-config={self.cnf_config}'] output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output.decode("utf-8")) try: - for namespace in ["cnf-testsuite", "litmus"]: + for namespace in ["cnf-testsuite", "cnf-default"]: self.corev1.delete_namespace(namespace) self.__logger.debug("delete_namespace: %s", namespace) except client.rest.ApiException: pass + # cnf-testsuite cleanup does not wait for removing reources + # use time.sleep(60) till + # https://github.com/cnti-testcatalog/testsuite/issues/2194 + # is fixed + time.sleep(60) @@ -5,7 +5,7 @@ envlist = pep8,pylint,yamllint,bashate,py310,bandit,perm usedevelop = True deps = -chttps://git.opnfv.org/functest/plain/upper-constraints.txt?h=stable/zed - -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/zed/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/unmaintained/zed/upper-constraints.txt} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt install_command = pip install --use-deprecated=legacy-resolver {opts} {packages} |