diff options
| -rw-r--r-- | behave_tests/__init__.py | 0 | ||||
| -rw-r--r-- | behave_tests/behavedriver.py | 70 | ||||
| -rw-r--r-- | behave_tests/features/__init__.py | 0 | ||||
| -rw-r--r-- | behave_tests/features/characterization-full.feature | 70 | ||||
| -rw-r--r-- | behave_tests/features/characterization-samples.feature | 44 | ||||
| -rw-r--r-- | behave_tests/features/environment.py | 62 | ||||
| -rw-r--r-- | behave_tests/features/non-regression.feature | 48 | ||||
| -rw-r--r-- | behave_tests/features/steps/__init__.py | 0 | ||||
| -rw-r--r-- | behave_tests/features/steps/steps.py | 455 | ||||
| -rw-r--r-- | docker/Dockerfile | 10 | ||||
| -rwxr-xr-x | docker/nfvbench-entrypoint.sh | 4 | ||||
| -rw-r--r-- | nfvbench/nfvbench.py | 3 | ||||
| -rw-r--r-- | nfvbench/utils.py | 18 | ||||
| -rw-r--r-- | requirements.txt | 4 | ||||
| -rw-r--r-- | setup.cfg | 3 | ||||
| -rw-r--r-- | test/mock_trex.py | 1 | ||||
| -rw-r--r-- | xtesting/ansible/host_vars/127.0.0.1 | 3 | ||||
| -rw-r--r-- | xtesting/ansible/site.yml | 4 | ||||
| -rw-r--r-- | xtesting/behaveframework.py | 123 | ||||
| -rw-r--r-- | xtesting/testcases.yaml | 62 |
20 files changed, 965 insertions, 19 deletions
diff --git a/behave_tests/__init__.py b/behave_tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/behave_tests/__init__.py diff --git a/behave_tests/behavedriver.py b/behave_tests/behavedriver.py new file mode 100644 index 0000000..ad0c8b7 --- /dev/null +++ b/behave_tests/behavedriver.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Copyright 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Define classes required to run any Behave test suites.""" + +from __future__ import division + +import json +import logging +import os + +from xtesting.core.behaveframework import BehaveFramework + +__author__ = "François-Régis Menguy <francoisregis.menguy@orange.com>" + + +class BehaveDriver(BehaveFramework): + """NFVbench custom BehaveDriver for Xtesting.""" + # pylint: disable=too-many-instance-attributes + + __logger = logging.getLogger('xtesting.core.behavedriver') + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.campaign_json_file = os.path.join(self.res_dir, 'campaign_result.json') + + def extract_nfvbench_results(self): + with open(self.campaign_json_file) as stream_: + self.details['results'] = json.load(stream_) + + def run(self, **kwargs): + + """Override existing Xtesting BehaveFramework core script run method + to extract NFVbench result and push them to DB + + Here are the steps: + * run Xtesting behave method: + * create the output directories if required, + * run behave features with parameters + * get the behave results in output.json, + * get the nfvbench results in campaign_result.json + + Args: + kwargs: Arbitrary keyword arguments. + + Returns: + EX_OK if all suites ran well. + EX_RUN_ERROR otherwise. + """ + try: + super().run(**kwargs) + self.extract_nfvbench_results() + self.__logger.info("NFVbench results were successfully parsed") + except Exception: # pylint: disable=broad-except + self.__logger.exception("Cannot parse NFVbench results") + return self.EX_RUN_ERROR + return self.EX_OK diff --git a/behave_tests/features/__init__.py b/behave_tests/features/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/behave_tests/features/__init__.py diff --git a/behave_tests/features/characterization-full.feature b/behave_tests/features/characterization-full.feature new file mode 100644 index 0000000..f8bca5f --- /dev/null +++ b/behave_tests/features/characterization-full.feature @@ -0,0 +1,70 @@ +@characterization +Feature: characterization + + @throughput + Scenario Outline: Run a NDR test for a defined frame size and flow count + Given 10 sec run duration + And TRex is restarted + And <frame_size> frame size + And <flow_count> flow count + And ndr rate + When NFVbench API is ready + Then 3 runs are started and waiting for maximum result + And push result to database + And extract offered rate result + + Examples: Frame sizes and flow counts + | frame_size | flow_count | + | 64 | 128 | + | 128 | 128 | + | 256 | 128 | + | 512 | 128 | + | 768 | 128 | + | 1024 | 128 | + | 1280 | 128 | + | 1518 | 128 | + | IMIX | 128 | + | 9000 | 128 | + | 64 | 10k | + | 128 | 10k | + | 256 | 10k | + | 512 | 10k | + | 768 | 10k | + | 1024 | 10k | + | 1280 | 10k | + | 1518 | 10k | + | IMIX | 10k | + | 9000 | 10k | + | 64 | 100k | + | 128 | 100k | + | 256 | 100k | + | 512 | 100k | + | 768 | 100k | + | 1024 | 100k | + | 1280 | 100k | + | 1518 | 100k | + | IMIX | 100k | + | 9000 | 100k | + + + @latency + Scenario Outline: Run a latency test for a defined frame size and throughput percentage + Given 10 sec run duration + And TRex is restarted + And <frame_size> frame size + And 100k flow count + And <throughput> rate of previous scenario + When NFVbench API is ready + Then run is started and waiting for result + And push result to database + + Examples: Frame sizes and throughput percentages + | frame_size | throughput | + | 64 | 70% | + | 64 | 90% | + | 768 | 70% | + | 768 | 90% | + | 1518 | 70% | + | 1518 | 90% | + | 9000 | 70% | + | 9000 | 90% | diff --git a/behave_tests/features/characterization-samples.feature b/behave_tests/features/characterization-samples.feature new file mode 100644 index 0000000..6adb184 --- /dev/null +++ b/behave_tests/features/characterization-samples.feature @@ -0,0 +1,44 @@ +@characterization +Feature: characterization + + @throughput + Scenario Outline: Run a NDR test for a defined frame size and flow count + Given 10 sec run duration + And TRex is restarted + And <frame_size> frame size + And <flow_count> flow count + And ndr rate + When NFVbench API is ready + Then 3 runs are started and waiting for maximum result + And push result to database + And extract offered rate result + + Examples: Frame sizes and flow counts + | frame_size | flow_count | + | 64 | 100k | + | 768 | 100k | + | 1518 | 100k | + | 9000 | 100k | + + + @latency + Scenario Outline: Run a latency test for a defined frame size and throughput percentage + Given 10 sec run duration + And TRex is restarted + And <frame_size> frame size + And 100k flow count + And <throughput> rate of previous scenario + When NFVbench API is ready + Then run is started and waiting for result + And push result to database + + Examples: Frame sizes and throughput percentages + | frame_size | throughput | + | 64 | 70% | + | 64 | 90% | + | 768 | 70% | + | 768 | 90% | + | 1518 | 70% | + | 1518 | 90% | + | 9000 | 70% | + | 9000 | 90% | diff --git a/behave_tests/features/environment.py b/behave_tests/features/environment.py new file mode 100644 index 0000000..ee1aa17 --- /dev/null +++ b/behave_tests/features/environment.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# Copyright 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import json +import os +import time + + +def before_all(context): + context.data = {'config': os.getenv('NFVBENCH_CONFIG_PATH', '/etc/nfvbench/nfvbench.cfg')} + + context.data['PROJECT_NAME'] = os.getenv('PROJECT_NAME', 'nfvbench') + context.data['TEST_DB_EXT_URL'] = os.getenv('TEST_DB_EXT_URL') + context.data['TEST_DB_URL'] = os.getenv('TEST_DB_URL') + context.data['BASE_TEST_DB_URL'] = '' + if context.data['TEST_DB_URL']: + context.data['BASE_TEST_DB_URL'] = context.data['TEST_DB_URL'].replace('results', '') + context.data['INSTALLER_TYPE'] = os.getenv('INSTALLER_TYPE') + context.data['DEPLOY_SCENARIO'] = os.getenv('DEPLOY_SCENARIO') + context.data['NODE_NAME'] = os.getenv('NODE_NAME', 'nfvbench') + context.data['BUILD_TAG'] = os.getenv('BUILD_TAG') + + +def before_feature(context, feature): + context.rates = {} + context.results = {} + context.start_time = time.time() + context.CASE_NAME = feature.name + + +def before_scenario(context, scenario): + context.tag = scenario.tags[0] + context.json = {'log_file': '/var/lib/xtesting/results/' + context.CASE_NAME + '/nfvbench.log'} + user_label = os.getenv('NFVBENCH_USER_LABEL', None) + if user_label: + context.json['user_label'] = user_label + loopvm_flavor = os.getenv('NFVBENCH_LOOPVM_FLAVOR_NAME', None) + if loopvm_flavor: + context.json['flavor_type'] = loopvm_flavor + context.synthesis = {} + + +def after_feature(context, feature): + if context.results: + with open(os.path.join( + '/var/lib/xtesting/results/' + context.CASE_NAME + '/campaign_result.json'), "w") \ + as outfile: + json.dump(context.results, outfile) diff --git a/behave_tests/features/non-regression.feature b/behave_tests/features/non-regression.feature new file mode 100644 index 0000000..5774418 --- /dev/null +++ b/behave_tests/features/non-regression.feature @@ -0,0 +1,48 @@ +@non-regression +Feature: non-regression + + @throughput + Scenario Outline: Run a NDR test for a defined frame size + Given 10 sec run duration + And TRex is restarted + And <frame_size> frame size + And 100k flow count + And ndr rate + When NFVbench API is ready + Then 3 runs are started and waiting for maximum result + And push result to database + And extract offered rate result + And verify throughput result is in same range as the previous result + And verify throughput result is in same range as the characterization result + + Examples: Frame sizes + | frame_size | + | 64 | + | 768 | + | 1518 | + | 9000 | + + + @latency + Scenario Outline: Run a latency test for a defined frame size and throughput percentage + Given 10 sec run duration + And TRex is restarted + And <frame_size> frame size + And 100k flow count + And <throughput> rate of previous scenario + When NFVbench API is ready + Then run is started and waiting for result + And push result to database + And verify latency result is in same range as the previous result + And verify latency result is in same range as the characterization result + + Examples: Frame sizes and throughput percentages + | frame_size | throughput | + | 64 | 70% | + | 64 | 90% | + | 768 | 70% | + | 768 | 90% | + | 1518 | 70% | + | 1518 | 90% | + | 9000 | 70% | + | 9000 | 90% | diff --git a/behave_tests/features/steps/__init__.py b/behave_tests/features/steps/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/behave_tests/features/steps/__init__.py diff --git a/behave_tests/features/steps/steps.py b/behave_tests/features/steps/steps.py new file mode 100644 index 0000000..b20a9cc --- /dev/null +++ b/behave_tests/features/steps/steps.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python +# Copyright 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from functools import reduce + +from behave import given +from behave import when +from behave import then +from requests import RequestException +from retry import retry +import json +import requests +import subprocess +from subprocess import DEVNULL + +from nfvbench.summarizer import Formatter +from nfvbench.traffic_gen.traffic_utils import parse_rate_str + +STATUS_ERROR = "ERROR" + +STATUS_OK = "OK" + + +"""Given steps.""" + + +@given('PROJECT_NAME: {project_name}') +def override_xtesting_project_name(context, project_name): + context.data['PROJECT_NAME'] = project_name + + +@given('TEST_DB_EXT_URL: {test_db_ext_url}') +def override_xtesting_test_db_ext_url(context, test_db_ext_url): + context.data['TEST_DB_EXT_URL'] = test_db_ext_url + + +@given('TEST_DB_URL: {test_db_url}') +def override_xtesting_test_db_url(context, test_db_url): + context.data['TEST_DB_URL'] = test_db_url + context.data['BASE_TEST_DB_URL'] = context.data['TEST_DB_URL'].replace('results', '') + + +@given('INSTALLER_TYPE: {installer_type}') +def override_xtesting_installer_type(context, installer_type): + context.data['INSTALLER_TYPE'] = installer_type + + +@given('DEPLOY_SCENARIO: {deploy_scenario}') +def override_xtesting_deploy_scenario(context, deploy_scenario): + context.data['DEPLOY_SCENARIO'] = deploy_scenario + + +@given('NODE_NAME: {node_name}') +def override_xtesting_node_name(context, node_name): + context.data['NODE_NAME'] = node_name + + +@given('BUILD_TAG: {build_tag}') +def override_xtesting_build_tag(context, build_tag): + context.data['BUILD_TAG'] = build_tag + + +@given('NFVbench config from file: {config_path}') +def init_config(context, config_path): + context.data['config'] = config_path + + +@given('a JSON NFVbench config') +def init_config_from_json(context): + context.json.update(json.loads(context.text)) + + +@given('log file: {log_file_path}') +def log_config(context, log_file_path): + context.json['log_file'] = log_file_path + + +@given('json file: {json_file_path}') +def json_config(context, json_file_path): + context.json['json'] = json_file_path + + +@given('no clean up') +def add_no_clean_up_flag(context): + context.json['no_cleanup'] = 'true' + + +@given('TRex is restarted') +def add_restart(context): + context.json['restart'] = 'true' + + +@given('{label} label') +def add_label(context, label): + context.json['label'] = label + + +@given('{frame_size} frame size') +def add_frame_size(context, frame_size): + context.json['frame_sizes'] = [frame_size] + + +@given('{flow_count} flow count') +def add_flow_count(context, flow_count): + context.json['flow_count'] = flow_count + + +@given('{rate} rate') +def add_rate(context, rate): + context.json['rate'] = rate + + +@given('{duration} sec run duration') +def add_duration(context, duration): + context.json['duration_sec'] = duration + + +@given('{percentage_rate} rate of previous scenario') +def add_percentage_rate(context, percentage_rate): + context.percentage_rate = percentage_rate + rate = percentage_previous_rate(context, percentage_rate) + context.json['rate'] = rate + + +"""When steps.""" + + +@when('NFVbench API is ready') +@when('NFVbench API is ready on host {host_ip}') +@when('NFVbench API is ready on host {host_ip} and port {port:d}') +def start_server(context, host_ip="127.0.0.1", port=7555): + context.host_ip = host_ip + context.port = port + try: + # check if API is already available + requests.get( + "http://{host_ip}:{port}/status".format(host_ip=context.host_ip, port=context.port)) + except RequestException: + cmd = ["nfvbench", "-c", context.data['config'], "--server"] + if host_ip != "127.0.0.1": + cmd.append("--host") + cmd.append(host_ip) + if port != 7555: + cmd.append("--port") + cmd.append(port) + + subprocess.Popen(cmd, stdout=DEVNULL, stderr=subprocess.STDOUT) + + test_nfvbench_api(context) + + +"""Then steps.""" + + +@then('run is started and waiting for result') +@then('{repeat:d} runs are started and waiting for maximum result') +def step_impl(context, repeat=1): + results = [] + if 'json' not in context.json: + context.json['json'] = '/var/lib/xtesting/results/' + context.CASE_NAME + \ + '/nfvbench-' + context.tag + '-fs_' + \ + context.json['frame_sizes'][0] + '-fc_' + \ + context.json['flow_count'] + '-rate_' + \ + context.json['rate'] + '.json' + json_base_name = context.json['json'] + for i in range(repeat): + if repeat > 1: + context.json['json'] = json_base_name.strip('.json') + '-' + str(i) + '.json' + + url = "http://{ip}:{port}/start_run".format(ip=context.host_ip, port=context.port) + payload = json.dumps(context.json) + r = requests.post(url, data=payload, headers={'Content-Type': 'application/json'}) + context.request_id = json.loads(r.text)["request_id"] + assert r.status_code == 200 + result = wait_result(context) + results.append(result) + assert result["status"] == STATUS_OK + + + context.result = reduce( + lambda x, y: x if extract_value(x, "total_tx_rate") > extract_value(y, + "total_tx_rate") else y, + results) + + total_tx_rate = extract_value(context.result, "total_tx_rate") + context.rates[context.json['frame_sizes'][0] + '_' + context.json['flow_count']] = total_tx_rate + overall = extract_value(context.result, "overall") + avg_delay_usec = extract_value(overall, "avg_delay_usec") + # create a synthesis with offered pps and latency values + context.synthesis['total_tx_rate'] = total_tx_rate + context.synthesis['avg_delay_usec'] = avg_delay_usec + + +@then('extract offered rate result') +def save_rate_result(context): + total_tx_rate = extract_value(context.result, "total_tx_rate") + context.rates[context.json['frame_sizes'][0] + '_' + context.json['flow_count']] = total_tx_rate + + +@then('verify throughput result is in same range as the previous result') +@then('verify throughput result is greater than {threshold} of the previous result') +def get_throughput_result_from_database(context, threshold='90%'): + last_result = get_last_result(context) + + if last_result: + compare_throughput_values(context, last_result, threshold) + + +@then('verify latency result is in same range as the previous result') +@then('verify latency result is greater than {threshold} of the previous result') +def get_latency_result_from_database(context, threshold='90%'): + last_result = get_last_result(context) + + if last_result: + compare_latency_values(context, last_result, threshold) + +@then( + 'verify result is in [{min_reference_value}pps, {max_reference_value}pps] range for throughput') +def compare_throughput_pps_result_with_range_values(context, min_reference_value, + max_reference_value): + context.unit = 'pps' + reference_values = [min_reference_value + 'pps', max_reference_value + 'pps'] + throughput_comparison(context, reference_values=reference_values) + + +@then( + 'verify result is in [{min_reference_value}bps, {max_reference_value}bps] range for throughput') +def compare_throughput_bps_result_with_range_values(context, min_reference_value, + max_reference_value): + context.unit = 'bps' + reference_values = [min_reference_value + 'bps', max_reference_value + 'bps'] + throughput_comparison(context, reference_values=reference_values) + + +@then('verify result is in {reference_values} range for latency') +def compare_result_with_range_values(context, reference_values): + latency_comparison(context, reference_values=reference_values) + + +@then('verify throughput result is in same range as the characterization result') +@then('verify throughput result is greater than {threshold} of the characterization result') +def get_characterization_throughput_result_from_database(context, threshold='90%'): + last_result = get_last_result(context, True) + if not last_result: + raise AssertionError("No characterization result found.") + compare_throughput_values(context, last_result, threshold) + + +@then('verify latency result is in same range as the characterization result') +@then('verify latency result is greater than {threshold} of the characterization result') +def get_characterization_latency_result_from_database(context, threshold='90%'): + last_result = get_last_result(context, True) + if not last_result: + raise AssertionError("No characterization result found.") + compare_latency_values(context, last_result, threshold) + +@then('push result to database') +def push_result_database(context): + if context.tag == "latency": + # override input rate value with percentage one to avoid no match + # if pps is not accurate with previous one + context.json["rate"] = context.percentage_rate + json_result = {"synthesis": context.synthesis, "input": context.json, "output": context.result} + + if context.tag not in context.results: + context.results[context.tag] = [json_result] + else: + context.results[context.tag].append(json_result) + + +"""Utils methods.""" + + +@retry(AssertionError, tries=10, delay=5.0, logger=None) +def test_nfvbench_api(context): + try: + r = requests.get("http://{ip}:{port}/status".format(ip=context.host_ip, port=context.port)) + assert r.status_code == 200 + assert json.loads(r.text)["error_message"] == "no pending NFVbench run" + except RequestException as exc: + raise AssertionError("Fail to access NFVbench API") from exc + + +@retry(AssertionError, tries=1000, delay=2.0, logger=None) +def wait_result(context): + r = requests.get("http://{ip}:{port}/status".format(ip=context.host_ip, port=context.port)) + context.raw_result = r.text + result = json.loads(context.raw_result) + assert r.status_code == 200 + assert result["status"] == STATUS_OK or result["status"] == STATUS_ERROR + return result + + +def percentage_previous_rate(context, rate): + previous_rate = context.rates[context.json['frame_sizes'][0] + '_' + context.json['flow_count']] + + if rate.endswith('%'): + rate_percent = convert_percentage_str_to_float(rate) + return str(int(previous_rate * rate_percent)) + 'pps' + raise Exception('Unknown rate string format %s' % rate) + + +def convert_percentage_str_to_float(percentage): + float_percent = float(percentage.replace('%', '').strip()) + if float_percent <= 0 or float_percent > 100.0: + raise Exception('%s is out of valid range (must be 1-100%%)' % percentage) + return float_percent / 100 + + +def compare_throughput_values(context, last_result, threshold): + assert last_result["output"]["status"] == context.result["status"] + if last_result["output"]["status"] == "OK": + old_throughput = extract_value(last_result["output"], "total_tx_rate") + throughput_comparison(context, old_throughput, threshold=threshold) + + +def compare_latency_values(context, last_result, threshold): + assert last_result["output"]["status"] == context.result["status"] + if last_result["output"]["status"] == "OK": + old_latency = extract_value(extract_value(last_result["output"], "overall"), + "avg_delay_usec") + latency_comparison(context, old_latency, threshold=threshold) + + +def throughput_comparison(context, old_throughput_pps=None, threshold=None, reference_values=None): + current_throughput_pps = extract_value(context.result, "total_tx_rate") + + if old_throughput_pps: + if not current_throughput_pps >= convert_percentage_str_to_float( + threshold) * old_throughput_pps: + raise AssertionError( + "Current run throughput {current_throughput_pps} is not over {threshold} " + " of previous value ({old_throughput_pps}pps)".format( + current_throughput_pps=Formatter.suffix('pps')( + Formatter.standard(current_throughput_pps)), + threshold=threshold, old_throughput_pps=Formatter.suffix('pps')( + Formatter.standard(old_throughput_pps)))) + elif reference_values: + if context.unit == 'bps': + current_throughput = extract_value(context.result, "offered_tx_rate_bps") + reference_values = [int(parse_rate_str(x)['rate_bps']) for x in reference_values] + formatted_current_throughput = Formatter.bits(current_throughput) + formatted_min_reference_value = Formatter.bits(reference_values[0]) + formatted_max_reference_value = Formatter.bits(reference_values[1]) + else: + current_throughput = current_throughput_pps + reference_values = [int(parse_rate_str(x)['rate_pps']) for x in reference_values] + formatted_current_throughput = Formatter.suffix('pps')( + Formatter.standard(current_throughput)) + formatted_min_reference_value = Formatter.suffix('pps')( + Formatter.standard(reference_values[0])) + formatted_max_reference_value = Formatter.suffix('pps')( + Formatter.standard(reference_values[1])) + if not reference_values[0] <= int(current_throughput) <= reference_values[1]: + raise AssertionError( + "Current run throughput {current_throughput} is not in reference values " + "[{min_reference_value}, {max_reference_value}]".format( + current_throughput=formatted_current_throughput, + min_reference_value=formatted_min_reference_value, + max_reference_value=formatted_max_reference_value)) + + +def latency_comparison(context, old_latency=None, threshold=None, reference_values=None): + overall = extract_value(context.result, "overall") + current_latency = extract_value(overall, "avg_delay_usec") + + if old_latency: + if not current_latency <= (2 - convert_percentage_str_to_float(threshold)) * old_latency: + threshold = str(200 - int(threshold.strip('%'))) + '%' + raise AssertionError( + "Current run latency {current_latency}usec is not less than {threshold} of " + "previous value ({old_latency}usec)".format( + current_latency=Formatter.standard(current_latency), threshold=threshold, + old_latency=Formatter.standard(old_latency))) + elif reference_values: + if not reference_values[0] <= current_latency <= reference_values[1]: + raise AssertionError( + "Current run latency {current_latency}usec is not in reference values " + "[{min_reference_value}, {max_reference_value}]".format( + current_latency=Formatter.standard(current_latency), + min_reference_value=Formatter.standard(reference_values[0]), + max_reference_value=Formatter.standard(reference_values[1]))) + + +def get_result_from_input_values(input, result): + # Select required keys (other keys can be not set or unconsistent between scenarios) + required_keys = ['duration_sec', 'frame_sizes', 'flow_count', 'rate'] + if 'user_label' in result: + required_keys.append('user_label') + if 'flavor_type' in result: + required_keys.append('flavor_type') + subset_input = dict((k, input[k]) for k in required_keys if k in input) + subset_result = dict((k, result[k]) for k in required_keys if k in result) + return subset_input == subset_result + + +def extract_value(obj, key): + """Pull all values of specified key from nested JSON.""" + arr = [] + + def extract(obj, arr, key): + """Recursively search for values of key in JSON tree.""" + if isinstance(obj, dict): + for k, v in obj.items(): + if k == key: + arr.append(v) + elif isinstance(v, (dict, list)): + extract(v, arr, key) + elif isinstance(obj, list): + for item in obj: + extract(item, arr, key) + return arr + + results = extract(obj, arr, key) + return results[0] + + +def get_last_result(context, reference=None, page=None): + if reference: + case_name = 'characterization' + else: + case_name = context.CASE_NAME + url = context.data['TEST_DB_URL'] + '?project={project_name}&case={case_name}'.format( + project_name=context.data['PROJECT_NAME'], case_name=case_name) + if context.data['INSTALLER_TYPE']: + url += '&installer={installer_name}'.format(installer_name=context.data['INSTALLER_TYPE']) + if context.data['NODE_NAME']: + url += '&pod={pod_name}'.format(pod_name=context.data['NODE_NAME']) + url += '&criteria=PASS' + if page: + url += '&page={page}'.format(page=page) + last_results = requests.get(url) + assert last_results.status_code == 200 + last_results = json.loads(last_results.text) + for result in last_results["results"]: + for tagged_result in result["details"]["results"][context.tag]: + if get_result_from_input_values(tagged_result["input"], context.json): + return tagged_result + if last_results["pagination"]["current_page"] < last_results["pagination"]["total_pages"]: + page = last_results["pagination"]["current_page"] + 1 + return get_last_result(context, page) + return None diff --git a/docker/Dockerfile b/docker/Dockerfile index be93d8e..8676bbd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,20 +32,22 @@ RUN apt-get update && apt-get install -y \ && python3 get-pip.py \ && pip3 install -U pbr \ && pip3 install -U setuptools \ - && cd / \ + && cd /opt \ # Note: do not clone with --depth 1 as it will cause pbr to fail extracting the nfvbench version # from the git tag && git clone https://gerrit.opnfv.org/gerrit/nfvbench \ - && cd /nfvbench && pip3 install -e . \ + && cd nfvbench && pip3 install -e . \ && wget -O nfvbenchvm-$VM_IMAGE_VER.qcow2 http://artifacts.opnfv.org/nfvbench/images/nfvbenchvm_centos-$VM_IMAGE_VER.qcow2 \ # Override Xtesting testcases.yaml file by NFVbench default one && cp xtesting/testcases.yaml /usr/local/lib/python3.8/dist-packages/xtesting/ci/testcases.yaml \ + # Temporary override waiting for PR approval : https://gerrit.opnfv.org/gerrit/c/functest-xtesting/+/72431 + && cp xtesting/behaveframework.py /usr/local/lib/python3.8/dist-packages/xtesting/core/behaveframework.py \ && python3 ./docker/cleanup_generators.py \ - && rm -rf /nfvbench/.git \ + && rm -rf /opt/nfvbench/.git \ && apt-get remove -y wget git python3-dev libpython3.8-dev gcc \ && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* ENV TREX_EXT_LIBS "/opt/trex/$TREX_VER/external_libs" -ENTRYPOINT ["/nfvbench/docker/nfvbench-entrypoint.sh"] +ENTRYPOINT ["/opt/nfvbench/docker/nfvbench-entrypoint.sh"] diff --git a/docker/nfvbench-entrypoint.sh b/docker/nfvbench-entrypoint.sh index 812816d..913e5ba 100755 --- a/docker/nfvbench-entrypoint.sh +++ b/docker/nfvbench-entrypoint.sh @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. # -if [ -z "$1" ] || ([ $1 != 'start_rest_server' ] && [ $1 != 'run_tests' ]); then +if [ -z "$1" ] || ([ $1 != 'start_rest_server' ] && [ $1 != 'run_tests' ] && [ $1 != 'zip_campaign' ]); then tail -f /dev/null elif [ $1 == 'run_tests' ]; then PARAMS="" @@ -22,6 +22,8 @@ elif [ $1 == 'run_tests' ]; then PARAMS+="$var " done eval "run_tests $PARAMS" +elif [ $1 == 'zip_campaign' ]; then + zip_campaign else PARAMS="--server" if [ -n "$HOST" ]; then diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py index 7acb783..bd86810 100644 --- a/nfvbench/nfvbench.py +++ b/nfvbench/nfvbench.py @@ -246,8 +246,7 @@ class NFVBench(object): config.cache_size = config.flow_count # The size must be capped to 10000 (where does this limit come from?) - if config.cache_size > 10000: - config.cache_size = 10000 + config.cache_size = min(config.cache_size, 10000) config.duration_sec = float(config.duration_sec) config.interval_sec = float(config.interval_sec) diff --git a/nfvbench/utils.py b/nfvbench/utils.py index 80a0817..512422d 100644 --- a/nfvbench/utils.py +++ b/nfvbench/utils.py @@ -127,11 +127,11 @@ def get_intel_pci(nic_slot=None, nic_ports=None): trex_base_dir = '/opt/trex' contents = os.listdir(trex_base_dir) trex_dir = os.path.join(trex_base_dir, contents[0]) - process = subprocess.Popen(['python', 'dpdk_setup_ports.py', '-s'], - cwd=trex_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - devices, _ = process.communicate() + with subprocess.Popen(['python', 'dpdk_setup_ports.py', '-s'], + cwd=trex_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as process: + devices, _ = process.communicate() except Exception: devices = '' @@ -147,10 +147,10 @@ def get_intel_pci(nic_slot=None, nic_ports=None): intf_name = glob.glob("/sys/bus/pci/devices/%s/net/*" % port[0]) if intf_name: intf_name = intf_name[0][intf_name[0].rfind('/') + 1:] - process = subprocess.Popen(['ip', '-o', '-d', 'link', 'show', intf_name], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - intf_info, _ = process.communicate() + with subprocess.Popen(['ip', '-o', '-d', 'link', 'show', intf_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as process: + intf_info, _ = process.communicate() if re.search('team_slave|bond_slave', intf_info.decode("utf-8")): device_ports_list[port[0].split('.')[0]]['busy'] = True for port in matches: diff --git a/requirements.txt b/requirements.txt index 732d807..b002c68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,6 @@ flask>=0.12 fluent-logger>=0.5.3 netaddr>=0.7.19 xtesting>=0.87.0 -hdrhistogram>=0.8.0
\ No newline at end of file +hdrhistogram>=0.8.0 +behave>=1.2.6 +retry>=0.9.2
\ No newline at end of file @@ -35,11 +35,14 @@ classifier = [files] packages = nfvbench + behave_tests [entry_points] console_scripts = nfvbench = nfvbench.nfvbench:main nfvbench_client = client.nfvbench_client:main +xtesting.testcase = + nfvbench_behaveframework = behave_tests.behavedriver:BehaveDriver [compile_catalog] directory = nfvbench/locale diff --git a/test/mock_trex.py b/test/mock_trex.py index ac7daf1..cb8a081 100644 --- a/test/mock_trex.py +++ b/test/mock_trex.py @@ -45,6 +45,7 @@ except ImportError: api_mod.Ether = STLDummy api_mod.ARP = STLDummy api_mod.IP = STLDummy + api_mod.ARP = STLDummy api_mod.STLClient = STLDummy api_mod.STLFlowLatencyStats = STLDummy api_mod.STLFlowStats = STLDummy diff --git a/xtesting/ansible/host_vars/127.0.0.1 b/xtesting/ansible/host_vars/127.0.0.1 index 9e3f1b9..125032f 100644 --- a/xtesting/ansible/host_vars/127.0.0.1 +++ b/xtesting/ansible/host_vars/127.0.0.1 @@ -1,5 +1,8 @@ docker_args: env: {} + params: + net: host + privileged: true volumes: - /lib/modules/$(uname -r):/lib/modules/$(uname -r) - /usr/src/kernels:/usr/src/kernels -v /dev:/dev diff --git a/xtesting/ansible/site.yml b/xtesting/ansible/site.yml index 4643a32..37fa6c3 100644 --- a/xtesting/ansible/site.yml +++ b/xtesting/ansible/site.yml @@ -16,5 +16,7 @@ - container: nfvbench tests: - 10kpps-pvp-run + - characterization + - non-regression properties: - execution-type: SEQUENTIALLY + execution-type: SEQUENTIALLY
\ No newline at end of file diff --git a/xtesting/behaveframework.py b/xtesting/behaveframework.py new file mode 100644 index 0000000..651240d --- /dev/null +++ b/xtesting/behaveframework.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# Copyright 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Define classes required to run any Behave test suites.""" + +from __future__ import division + +import logging +import os +import time + +import json +import six + +from behave.__main__ import main as behave_main + +from xtesting.core import testcase + +__author__ = "Deepak Chandella <deepak.chandella@orange.com>" + + +class BehaveFramework(testcase.TestCase): + """BehaveFramework runner.""" + # pylint: disable=too-many-instance-attributes + + __logger = logging.getLogger(__name__) + dir_results = "/var/lib/xtesting/results" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.json_file = os.path.join(self.res_dir, 'output.json') + self.total_tests = 0 + self.pass_tests = 0 + self.fail_tests = 0 + self.skip_tests = 0 + self.response = None + + def parse_results(self): + """Parse output.json and get the details in it.""" + with open(self.json_file) as stream_: + self.response = json.load(stream_) + if self.response: + self.total_tests = len(self.response) + for item in self.response: + if item['status'] == 'passed': + self.pass_tests += 1 + elif item['status'] == 'failed': + self.fail_tests += 1 + elif item['status'] == 'skipped': + self.skip_tests += 1 + self.result = 100 * ( + self.pass_tests / self.total_tests) + self.details = {} + self.details['total_tests'] = self.total_tests + self.details['pass_tests'] = self.pass_tests + self.details['fail_tests'] = self.fail_tests + self.details['skip_tests'] = self.skip_tests + self.details['tests'] = self.response + + def run(self, **kwargs): + """Run the BehaveFramework feature files + + Here are the steps: + * create the output directories if required, + * run behave features with parameters + * get the results in output.json, + + Args: + kwargs: Arbitrary keyword arguments. + + Returns: + EX_OK if all suites ran well. + EX_RUN_ERROR otherwise. + """ + try: + suites = kwargs["suites"] + tags = kwargs.get("tags", []) + console = kwargs["console"] if "console" in kwargs else False + except KeyError: + self.__logger.exception("Mandatory args were not passed") + return self.EX_RUN_ERROR + if not os.path.exists(self.res_dir): + try: + os.makedirs(self.res_dir) + except Exception: # pylint: disable=broad-except + self.__logger.exception("Cannot create %s", self.res_dir) + return self.EX_RUN_ERROR + config = ['--tags=' + ','.join(tags), + '--junit', '--junit-directory={}'.format(self.res_dir), + '--format=json', '--outfile={}'.format(self.json_file)] + if six.PY3: + html_file = os.path.join(self.res_dir, 'output.html') + config += ['--format=behave_html_formatter:HTMLFormatter', + '--outfile={}'.format(html_file)] + if console: + config += ['--format=pretty', + '--outfile=-'] + for feature in suites: + config.append(feature) + self.start_time = time.time() + behave_main(config) + self.stop_time = time.time() + + try: + self.parse_results() + self.__logger.info("Results were successfully parsed") + except Exception: # pylint: disable=broad-except + self.__logger.exception("Cannot parse results") + return self.EX_RUN_ERROR + return self.EX_OK diff --git a/xtesting/testcases.yaml b/xtesting/testcases.yaml index cbb5c45..eea60b9 100644 --- a/xtesting/testcases.yaml +++ b/xtesting/testcases.yaml @@ -6,6 +6,65 @@ tiers: description: 'Data Plane Performance Testing' testcases: - + case_name: characterization + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'nfvbench_behaveframework' + args: + suites: + - /opt/nfvbench/behave_tests/features/characterization-full.feature + tags: + - characterization + console: + - true + - + case_name: non-regression + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'nfvbench_behaveframework' + args: + suites: + - /opt/nfvbench/behave_tests/features/non-regression.feature + tags: + - non-regression + console: + - true + + - + name: nfvbench-rapid-characterization + order: 2 + description: 'Data Plane Performance Testing' + testcases: + - + case_name: rapid-characterization + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'nfvbench_behaveframework' + args: + suites: + - /opt/nfvbench/behave_tests/features/characterization-samples.feature + tags: + - characterization + console: + - true + - + name: nfvbench-demo + order: 3 + description: 'Data Plane Performance Testing' + testcases: + - case_name: 10kpps-pvp-run project_name: nfvbench criteria: 100 @@ -17,4 +76,5 @@ tiers: args: cmd: - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps - + console: + - true
\ No newline at end of file |