diff options
| -rw-r--r-- | behave_tests/features/quick-test-10kpps.feature | 13 | ||||
| -rw-r--r-- | behave_tests/features/steps/steps.py | 2 | ||||
| -rw-r--r-- | docker/Dockerfile | 2 | ||||
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | xtesting/behaveframework.py | 123 | ||||
| -rw-r--r-- | xtesting/testcases.yaml | 23 |
6 files changed, 37 insertions, 128 deletions
diff --git a/behave_tests/features/quick-test-10kpps.feature b/behave_tests/features/quick-test-10kpps.feature new file mode 100644 index 0000000..d46000c --- /dev/null +++ b/behave_tests/features/quick-test-10kpps.feature @@ -0,0 +1,13 @@ +@quick-test-10kpps +Feature: quick-test-10kpps + + @throughput + Scenario: Run a 10s test at 10kpps with 64-byte frames and 128 flows + Given 10 sec run duration + And TRex is restarted + And 64 frame size + And 128 flow count + And 10kpps rate + When NFVbench API is ready + Then 1 runs are started and waiting for maximum result + And push result to database diff --git a/behave_tests/features/steps/steps.py b/behave_tests/features/steps/steps.py index b20a9cc..f4dda58 100644 --- a/behave_tests/features/steps/steps.py +++ b/behave_tests/features/steps/steps.py @@ -284,7 +284,7 @@ def push_result_database(context): """Utils methods.""" -@retry(AssertionError, tries=10, delay=5.0, logger=None) +@retry(AssertionError, tries=24, delay=5.0, logger=None) def test_nfvbench_api(context): try: r = requests.get("http://{ip}:{port}/status".format(ip=context.host_ip, port=context.port)) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4336261..e851c76 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,8 +40,6 @@ RUN apt-get update && apt-get install -y \ && wget -O nfvbenchvm-$VM_IMAGE_VER.qcow2 http://artifacts.opnfv.org/nfvbench/images/nfvbenchvm_centos-$VM_IMAGE_VER.qcow2 \ # Override Xtesting testcases.yaml file by NFVbench default one && cp xtesting/testcases.yaml /usr/local/lib/python3.8/dist-packages/xtesting/ci/testcases.yaml \ - # Temporary override waiting for PR approval : https://gerrit.opnfv.org/gerrit/c/functest-xtesting/+/72431 - && cp xtesting/behaveframework.py /usr/local/lib/python3.8/dist-packages/xtesting/core/behaveframework.py \ && python3 ./docker/cleanup_generators.py \ && rm -rf /opt/nfvbench/.git \ # Symlink for retrocompatibility 4.x diff --git a/requirements.txt b/requirements.txt index 7c74119..a333380 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ tabulate>=0.7.5 flask>=0.12 fluent-logger>=0.5.3 netaddr>=0.7.19 -xtesting>=0.87.0 +xtesting>=0.92.0 hdrhistogram>=0.8.0 behave>=1.2.6 retry>=0.9.2
\ No newline at end of file diff --git a/xtesting/behaveframework.py b/xtesting/behaveframework.py deleted file mode 100644 index 651240d..0000000 --- a/xtesting/behaveframework.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# Copyright 2021 Orange -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -"""Define classes required to run any Behave test suites.""" - -from __future__ import division - -import logging -import os -import time - -import json -import six - -from behave.__main__ import main as behave_main - -from xtesting.core import testcase - -__author__ = "Deepak Chandella <deepak.chandella@orange.com>" - - -class BehaveFramework(testcase.TestCase): - """BehaveFramework runner.""" - # pylint: disable=too-many-instance-attributes - - __logger = logging.getLogger(__name__) - dir_results = "/var/lib/xtesting/results" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.json_file = os.path.join(self.res_dir, 'output.json') - self.total_tests = 0 - self.pass_tests = 0 - self.fail_tests = 0 - self.skip_tests = 0 - self.response = None - - def parse_results(self): - """Parse output.json and get the details in it.""" - with open(self.json_file) as stream_: - self.response = json.load(stream_) - if self.response: - self.total_tests = len(self.response) - for item in self.response: - if item['status'] == 'passed': - self.pass_tests += 1 - elif item['status'] == 'failed': - self.fail_tests += 1 - elif item['status'] == 'skipped': - self.skip_tests += 1 - self.result = 100 * ( - self.pass_tests / self.total_tests) - self.details = {} - self.details['total_tests'] = self.total_tests - self.details['pass_tests'] = self.pass_tests - self.details['fail_tests'] = self.fail_tests - self.details['skip_tests'] = self.skip_tests - self.details['tests'] = self.response - - def run(self, **kwargs): - """Run the BehaveFramework feature files - - Here are the steps: - * create the output directories if required, - * run behave features with parameters - * get the results in output.json, - - Args: - kwargs: Arbitrary keyword arguments. - - Returns: - EX_OK if all suites ran well. - EX_RUN_ERROR otherwise. - """ - try: - suites = kwargs["suites"] - tags = kwargs.get("tags", []) - console = kwargs["console"] if "console" in kwargs else False - except KeyError: - self.__logger.exception("Mandatory args were not passed") - return self.EX_RUN_ERROR - if not os.path.exists(self.res_dir): - try: - os.makedirs(self.res_dir) - except Exception: # pylint: disable=broad-except - self.__logger.exception("Cannot create %s", self.res_dir) - return self.EX_RUN_ERROR - config = ['--tags=' + ','.join(tags), - '--junit', '--junit-directory={}'.format(self.res_dir), - '--format=json', '--outfile={}'.format(self.json_file)] - if six.PY3: - html_file = os.path.join(self.res_dir, 'output.html') - config += ['--format=behave_html_formatter:HTMLFormatter', - '--outfile={}'.format(html_file)] - if console: - config += ['--format=pretty', - '--outfile=-'] - for feature in suites: - config.append(feature) - self.start_time = time.time() - behave_main(config) - self.stop_time = time.time() - - try: - self.parse_results() - self.__logger.info("Results were successfully parsed") - except Exception: # pylint: disable=broad-except - self.__logger.exception("Cannot parse results") - return self.EX_RUN_ERROR - return self.EX_OK diff --git a/xtesting/testcases.yaml b/xtesting/testcases.yaml index bb4a521..03579b7 100644 --- a/xtesting/testcases.yaml +++ b/xtesting/testcases.yaml @@ -60,8 +60,29 @@ tiers: console: - true - - name: nfvbench-demo + name: quick-test-10kpps order: 3 + description: 'Quick nfvbench test at low packet rate' + testcases: + - + case_name: quick-test-10kpps + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'nfvbench_behaveframework' + args: + suites: + - /opt/nfvbench/behave_tests/features/quick-test-10kpps.feature + tags: + - quick-test-10kpps + console: + - true + - + name: nfvbench-demo + order: 4 description: 'Data Plane Performance Testing' testcases: - |