diff options
| -rw-r--r-- | docs/development/design/ndrpdr.rst | 10 | ||||
| -rw-r--r-- | nfvbench/chain_clients.py | 7 | ||||
| -rw-r--r-- | nfvbench/chain_managers.py | 2 | ||||
| -rw-r--r-- | nfvbench/compute.py | 17 | ||||
| -rw-r--r-- | nfvbench/config.py | 2 | ||||
| -rw-r--r-- | nfvbench/nfvbench.py | 4 | ||||
| -rw-r--r-- | nfvbench/summarizer.py | 3 | ||||
| -rw-r--r-- | nfvbench/traffic_client.py | 21 | ||||
| -rw-r--r-- | nfvbench/traffic_gen/dummy.py | 98 | ||||
| -rw-r--r-- | nfvbench/traffic_gen/traffic_utils.py | 32 | ||||
| -rw-r--r-- | nfvbench/utils.py | 2 | ||||
| -rw-r--r-- | test/test_nfvbench.py | 408 | ||||
| -rw-r--r-- | tox.ini | 2 |
13 files changed, 292 insertions, 316 deletions
diff --git a/docs/development/design/ndrpdr.rst b/docs/development/design/ndrpdr.rst index 4f611a0..5361174 100644 --- a/docs/development/design/ndrpdr.rst +++ b/docs/development/design/ndrpdr.rst @@ -28,7 +28,17 @@ The default value of 0.1 indicates for example that the measured NDR and PDR are actual NDR/PDR (e.g. 0.1% of 10Gbps is 10Mbps). It also determines how small the search range must be in the binary search. The recursion narrows down the range by half and stops when: + - the range is smaller than the configured load_epsilon value - or when the search hits 100% or 0% of line rate +One particularity of using a software traffic generator is that the requested Tx rate may not always be met due to +resource limitations (e.g. CPU is not fast enough to generate a very high load). The algorithm should take this into +consideration: + +- always monitor the actual Tx rate achieved +- actual Tx rate is always <= requested Tx rate +- the measured drop rate should always be relative to the actual Tx rate +- if the actual Tx rate is < requested Tx rate and the measured drop rate is already within threshold (<NDR/PDR threshold) then the binary search must stop with proper warning + diff --git a/nfvbench/chain_clients.py b/nfvbench/chain_clients.py index 57b15ee..fa21359 100644 --- a/nfvbench/chain_clients.py +++ b/nfvbench/chain_clients.py @@ -18,13 +18,12 @@ import os import re import time -import compute -from log import LOG - from glanceclient.v2 import client as glanceclient from neutronclient.neutron import client as neutronclient from novaclient.client import Client +import compute +from log import LOG class StageClientException(Exception): pass @@ -109,7 +108,7 @@ class BasicStageClient(object): phys1=network['provider:physical_network'], phys2=physical_network)) - LOG.info('Reusing existing network: ' + name) + LOG.info('Reusing existing network: %s', name) network['is_reuse'] = True return network diff --git a/nfvbench/chain_managers.py b/nfvbench/chain_managers.py index cbc53e2..8b605aa 100644 --- a/nfvbench/chain_managers.py +++ b/nfvbench/chain_managers.py @@ -111,7 +111,7 @@ class StatsManager(object): def _generate_traffic(self): if self.config.no_traffic: - return + return {} self.interval_collector = IntervalCollector(time.time()) self.interval_collector.attach_notifier(self.notifier) diff --git a/nfvbench/compute.py b/nfvbench/compute.py index 575744c..af1a0d6 100644 --- a/nfvbench/compute.py +++ b/nfvbench/compute.py @@ -16,16 +16,15 @@ import os import time import traceback -import keystoneauth1 -from log import LOG -import novaclient - from glanceclient import exc as glance_exception - try: from glanceclient.openstack.common.apiclient.exceptions import NotFound as GlanceImageNotFound except ImportError: from glanceclient.v1.apiclient.exceptions import NotFound as GlanceImageNotFound +import keystoneauth1 +import novaclient + +from log import LOG class Compute(object): @@ -75,7 +74,7 @@ class Compute(object): "image at the specified location %s is correct.", image_file) return False except keystoneauth1.exceptions.http.NotFound as exc: - LOG.error("Authentication error while uploading the image:" + str(exc)) + LOG.error("Authentication error while uploading the image: %s", str(exc)) return False except Exception: LOG.error(traceback.format_exc()) @@ -258,7 +257,7 @@ class Compute(object): if hyp.host == host: return self.normalize_az_host(hyp.zone, host) # no match on host - LOG.error('Passed host name does not exist: ' + host) + LOG.error('Passed host name does not exist: %s', host) return None if self.config.availability_zone: return self.normalize_az_host(None, host) @@ -290,7 +289,7 @@ class Compute(object): return az_host # else continue - another zone with same host name? # no match - LOG.error('No match for availability zone and host ' + az_host) + LOG.error('No match for availability zone and host %s', az_host) return None else: return self.auto_fill_az(host_list, az_host) @@ -348,7 +347,7 @@ class Compute(object): if not self.config.availability_zone: LOG.error('Availability_zone must be configured') elif host_list: - LOG.error('No host matching the selection for availability zone: ' + + LOG.error('No host matching the selection for availability zone: %s', self.config.availability_zone) avail_list = [] else: diff --git a/nfvbench/config.py b/nfvbench/config.py index 8139389..5feeda5 100644 --- a/nfvbench/config.py +++ b/nfvbench/config.py @@ -14,9 +14,9 @@ # from attrdict import AttrDict -from log import LOG import yaml +from log import LOG def config_load(file_name, from_cfg=None, whitelist_keys=None): """Load a yaml file into a config dict, merge with from_cfg if not None diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py index 4c9f56c..6f59e24 100644 --- a/nfvbench/nfvbench.py +++ b/nfvbench/nfvbench.py @@ -481,11 +481,11 @@ def main(): # override default config options with start config at path parsed from CLI # check if it is an inline yaml/json config or a file name if os.path.isfile(opts.config): - LOG.info('Loading configuration file: ' + opts.config) + LOG.info('Loading configuration file: %s', opts.config) config = config_load(opts.config, config, whitelist_keys) config.name = os.path.basename(opts.config) else: - LOG.info('Loading configuration string: ' + opts.config) + LOG.info('Loading configuration string: %s', opts.config) config = config_loads(opts.config, config, whitelist_keys) # traffic profile override options diff --git a/nfvbench/summarizer.py b/nfvbench/summarizer.py index 70ad389..1676e93 100644 --- a/nfvbench/summarizer.py +++ b/nfvbench/summarizer.py @@ -20,9 +20,10 @@ import math import bitmath import pytz -from specs import ChainType from tabulate import tabulate +from specs import ChainType + class Formatter(object): """Collection of string formatter methods""" diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py index a1c4954..8959cab 100644 --- a/nfvbench/traffic_client.py +++ b/nfvbench/traffic_client.py @@ -67,6 +67,9 @@ class TrafficRunner(object): def poll_stats(self): if not self.is_running(): return None + if self.client.skip_sleep: + self.stop() + return self.client.get_stats() time_elapsed = self.time_elapsed() if time_elapsed > self.duration_sec: self.stop() @@ -102,10 +105,10 @@ class IpBlock(object): '''Reserve a range of count consecutive IP addresses spaced by step ''' if self.next_free + count > self.max_available: - raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d', - self.next_free, - self.max_available, - count) + raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d' % + (self.next_free, + self.max_available, + count)) first_ip = self.get_ip(self.next_free) last_ip = self.get_ip(self.next_free + count - 1) self.next_free += count @@ -393,7 +396,7 @@ class TrafficGeneratorFactory(object): class TrafficClient(object): PORTS = [0, 1] - def __init__(self, config, notifier=None): + def __init__(self, config, notifier=None, skip_sleep=False): generator_factory = TrafficGeneratorFactory(config) self.gen = generator_factory.get_generator_client() self.tool = generator_factory.get_tool() @@ -414,6 +417,8 @@ class TrafficClient(object): self.current_total_rate = {'rate_percent': '10'} if self.config.single_run: self.current_total_rate = utils.parse_rate_str(self.config.rate) + # UT with dummy TG can bypass all sleeps + self.skip_sleep = skip_sleep def set_macs(self): for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices): @@ -461,7 +466,8 @@ class TrafficClient(object): self.gen.clear_stats() self.gen.start_traffic() LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count) - time.sleep(self.config.generic_poll_sec) + if not self.skip_sleep: + time.sleep(self.config.generic_poll_sec) self.gen.stop_traffic() stats = self.gen.get_stats() @@ -481,7 +487,8 @@ class TrafficClient(object): LOG.info('End-to-end connectivity ensured') return - time.sleep(self.config.generic_poll_sec) + if not self.skip_sleep: + time.sleep(self.config.generic_poll_sec) raise TrafficClientException('End-to-end connectivity cannot be ensured') diff --git a/nfvbench/traffic_gen/dummy.py b/nfvbench/traffic_gen/dummy.py index d8c01e9..b43030f 100644 --- a/nfvbench/traffic_gen/dummy.py +++ b/nfvbench/traffic_gen/dummy.py @@ -13,6 +13,7 @@ # under the License. from traffic_base import AbstractTrafficGenerator +import traffic_utils as utils class DummyTG(AbstractTrafficGenerator): @@ -22,10 +23,13 @@ class DummyTG(AbstractTrafficGenerator): Useful for unit testing without actually generating any traffic. """ - def __init__(self, runner): - AbstractTrafficGenerator.__init__(self, runner) + def __init__(self, config): + AbstractTrafficGenerator.__init__(self, config) self.port_handle = [] self.rates = [] + self.l2_frame_size = 0 + self.duration_sec = self.config.duration_sec + self.intf_speed = config.generator_config.intf_speed def get_version(self): return "0.1" @@ -33,6 +37,59 @@ class DummyTG(AbstractTrafficGenerator): def init(self): pass + def get_tx_pps_dropped_pps(self, tx_rate): + '''Get actual tx packets based on requested tx rate + + :param tx_rate: requested TX rate with unit ('40%', '1Mbps', '1000pps') + + :return: the actual TX pps and the dropped pps corresponding to the requested TX rate + ''' + dr, tx = self.__get_dr_actual_tx(tx_rate) + actual_tx_bps = utils.load_to_bps(tx, self.intf_speed) + avg_packet_size = utils.get_average_packet_size(self.l2_frame_size) + tx_packets = utils.bps_to_pps(actual_tx_bps, avg_packet_size) + + dropped = tx_packets * dr / 100 + # print '===get_tx_pkts_dropped_pkts req tex=', tx_rate, 'dr=', dr, + # 'actual tx rate=', tx, 'actual tx pkts=', tx_packets, 'dropped=', dropped + return int(tx_packets), int(dropped) + + def set_response_curve(self, lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100): + '''Set traffic gen response characteristics + + Specifies the drop rate curve and the actual TX curve + :param float lr_dr: The actual drop rate at TX line rate (in %, 0..100) + :param float ndr: The true NDR (0 packet drop) in % (0..100) of line rate" + :param float max_actual_tx: highest actual TX when requested TX is 100% + :param float max_11_tx: highest requested TX that results in same actual TX + ''' + self.target_ndr = ndr + if ndr < 100: + self.dr_slope = float(lr_dr) / (100 - ndr) + else: + self.dr_slope = 0 + self.max_11_tx = max_11_tx + self.max_actual_tx = max_actual_tx + if max_11_tx < 100: + self.tx_slope = float(max_actual_tx - max_11_tx) / (100 - max_11_tx) + else: + self.tx_slope = 0 + + def __get_dr_actual_tx(self, requested_tx_rate): + '''Get drop rate at given requested tx rate + :param float requested_tx_rate: requested tx rate in % (0..100) + :return: the drop rate and actual tx rate at that requested_tx_rate in % (0..100) + ''' + if requested_tx_rate <= self.max_11_tx: + actual_tx = requested_tx_rate + else: + actual_tx = self.max_11_tx + (requested_tx_rate - self.max_11_tx) * self.tx_slope + if actual_tx <= self.target_ndr: + dr = 0.0 + else: + dr = (actual_tx - self.target_ndr) * self.dr_slope + return dr, actual_tx + def connect(self): ports = list(self.config.generator_config.ports) self.port_handle = ports @@ -44,32 +101,57 @@ class DummyTG(AbstractTrafficGenerator): pass def create_traffic(self, l2frame_size, rates, bidirectional, latency=True): - pass + self.rates = [utils.to_rate_str(rate) for rate in rates] + self.l2_frame_size = l2frame_size def clear_streamblock(self): pass def get_stats(self): + '''Get stats from current run. + + The binary search mainly looks at 2 results to make the decision: + actual tx packets + actual rx dropped packets + From the Requested TX rate - we get the Actual TX rate and the RX drop rate + From the Run duration and actual TX rate - we get the actual total tx packets + From the Actual tx packets and RX drop rate - we get the RX dropped packets + ''' result = {} - for ph in self.port_handle: + total_tx_pps = 0 + + # use dummy values for all other result field as the goal is to + # test the ndr/pdr convergence code + for idx, ph in enumerate(self.port_handle): + requested_tx_rate = utils.get_load_from_rate(self.rates[idx]) + tx_pps, dropped_pps = self.get_tx_pps_dropped_pps(requested_tx_rate) + + # total packets sent per direction - used by binary search + total_pkts = tx_pps * self.duration_sec + dropped_pkts = dropped_pps * self.duration_sec + _, tx_pkt_rate = self.__get_dr_actual_tx(requested_tx_rate) result[ph] = { 'tx': { - 'total_pkts': 1000, + 'total_pkts': total_pkts, 'total_pkt_bytes': 100000, - 'pkt_rate': 100, + 'pkt_rate': tx_pkt_rate, 'pkt_bit_rate': 1000000 }, 'rx': { - 'total_pkts': 1000, + # total packets received + 'total_pkts': total_pkts - dropped_pkts, 'total_pkt_bytes': 100000, 'pkt_rate': 100, 'pkt_bit_rate': 1000000, - 'dropped_pkts': 0 + 'dropped_pkts': dropped_pkts } } result[ph]['rx']['max_delay_usec'] = 10.0 result[ph]['rx']['min_delay_usec'] = 1.0 result[ph]['rx']['avg_delay_usec'] = 2.0 + total_tx_pps += tx_pps + # actual total tx rate in pps + result['total_tx_rate'] = total_tx_pps return result def clear_stats(self): diff --git a/nfvbench/traffic_gen/traffic_utils.py b/nfvbench/traffic_gen/traffic_utils.py index e618c28..4a7f855 100644 --- a/nfvbench/traffic_gen/traffic_utils.py +++ b/nfvbench/traffic_gen/traffic_utils.py @@ -75,6 +75,13 @@ def weighted_avg(weight, count): return sum([x[0] * x[1] for x in zip(weight, count)]) / sum(weight) return float('nan') +def _get_bitmath_rate(rate_bps): + rate = rate_bps.replace('ps', '').strip() + bitmath_rate = bitmath.parse_string(rate) + if bitmath_rate.bits <= 0: + raise Exception('%s is out of valid range' % rate_bps) + return bitmath_rate + def parse_rate_str(rate_str): if rate_str.endswith('pps'): rate_pps = rate_str[:-3] @@ -103,6 +110,26 @@ def parse_rate_str(rate_str): else: raise Exception('Unknown rate string format %s' % rate_str) +def get_load_from_rate(rate_str, avg_frame_size=64, line_rate='10Gbps'): + '''From any rate string (with unit) return the corresponding load (in % unit) + + :param str rate_str: the rate to convert - must end with a unit (e.g. 1Mpps, 30%, 1Gbps) + :param int avg_frame_size: average frame size in bytes (needed only if pps is given) + :param str line_rate: line rate ending with bps unit (e.g. 1Mbps, 10Gbps) is the rate that + corresponds to 100% rate + :return float: the corresponding rate in % of line rate + ''' + rate_dict = parse_rate_str(rate_str) + if 'rate_percent' in rate_dict: + return float(rate_dict['rate_percent']) + lr_bps = _get_bitmath_rate(line_rate).bits + if 'rate_bps' in rate_dict: + bps = int(rate_dict['rate_bps']) + else: + # must be rate_pps + pps = rate_dict['rate_pps'] + bps = pps_to_bps(pps, avg_frame_size) + return bps_to_load(bps, lr_bps) def divide_rate(rate, divisor): if 'rate_pps' in rate: @@ -130,8 +157,9 @@ def to_rate_str(rate): elif 'rate_percent' in rate: load = rate['rate_percent'] return '{}%'.format(load) - else: - assert False + assert False + # avert pylint warning + return None def nan_replace(d): diff --git a/nfvbench/utils.py b/nfvbench/utils.py index 412dfae..20dc588 100644 --- a/nfvbench/utils.py +++ b/nfvbench/utils.py @@ -61,7 +61,7 @@ def save_json_result(result, json_file, std_json_path, service_chain, service_ch if filepaths: for file_path in filepaths: - LOG.info('Saving results in json file: ' + file_path + "...") + LOG.info('Saving results in json file: %s...', file_path) with open(file_path, 'w') as jfp: json.dump(result, jfp, diff --git a/test/test_nfvbench.py b/test/test_nfvbench.py index 2578407..05490e7 100644 --- a/test/test_nfvbench.py +++ b/test/test_nfvbench.py @@ -18,6 +18,8 @@ import logging import os import sys +import pytest + from attrdict import AttrDict from nfvbench.config import config_loads from nfvbench.credentials import Credentials @@ -28,7 +30,6 @@ from nfvbench.network import Network from nfvbench.specs import ChainType from nfvbench.specs import Encaps import nfvbench.traffic_gen.traffic_utils as traffic_utils -import pytest __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -267,183 +268,6 @@ def test_pvp_chain_run(pvp_chain): """ # ========================================================================= -# PVVP Chain tests -# ========================================================================= - -""" -@pytest.fixture -def pvvp_chain(monkeypatch, openstack_vxlan_spec): - tor_vni1 = Interface('vni-4097', 'n9k', 50, 77) - vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48) - vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77) - vsw_vif3 = Interface('VirtualEthernet0/0/0', 'vpp', 77, 47) - vsw_vif4 = Interface('VirtualEthernet0/0/1', 'vpp', 45, 77) - vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 44) - vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77) - tor_vni2 = Interface('vni-4098', 'n9k', 77, 40) - - def mock_init(self, *args, **kwargs): - self.vni_ports = [4099, 4100] - self.v2vnet = V2VNetwork() - self.specs = openstack_vxlan_spec - self.clients = { - 'vpp': AttrDict({ - 'get_v2v_network': lambda reverse=None: Network([vsw_vif3, vsw_vif4], reverse), - 'set_interface_counters': lambda pvvp=None: None, - 'set_v2v_counters': lambda: None, - }) - } - self.worker = AttrDict({ - 'run': lambda: None, - }) - - def mock_empty(self, *args, **kwargs): - pass - - def mock_get_network(self, traffic_port, vni_id, reverse=False): - if vni_id == 0: - return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse) - else: - return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse) - - def mock_get_data(self): - return {} - - monkeypatch.setattr(PVVPChain, '_get_network', mock_get_network) - monkeypatch.setattr(PVVPChain, '_get_data', mock_get_data) - monkeypatch.setattr(PVVPChain, '_setup', mock_empty) - monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty) - monkeypatch.setattr(PVVPChain, '_generate_traffic', mock_empty) - monkeypatch.setattr(PVVPChain, '__init__', mock_init) - - return PVVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None) - - -def test_pvvp_chain_run(pvvp_chain): - result = pvvp_chain.run() - - expected_result = { - 'raw_data': {}, - 'stats': None, - 'packet_analysis': - {'direction-forward': [ - OrderedDict([ - ('interface', 'vni-4097'), - ('device', 'n9k'), - ('packet_count', 50) - ]), - OrderedDict([ - ('interface', 'vxlan_tunnel0'), - ('device', 'vpp'), - ('packet_count', 48), - ('packet_drop_count', 2), - ('packet_drop_percentage', 4.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/2'), - ('device', 'vpp'), - ('packet_count', 48), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/0'), - ('device', 'vpp'), - ('packet_count', 47), - ('packet_drop_count', 1), - ('packet_drop_percentage', 2.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/1'), - ('device', 'vpp'), - ('packet_count', 45), - ('packet_drop_count', 2), - ('packet_drop_percentage', 4.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/3'), - ('device', 'vpp'), - ('packet_count', 44), - ('packet_drop_count', 1), - ('packet_drop_percentage', 2.0) - ]), - OrderedDict([ - ('interface', 'vxlan_tunnel1'), - ('device', 'vpp'), - ('packet_count', 43), - ('packet_drop_count', 1), - ('packet_drop_percentage', 2.0) - ]), - OrderedDict([ - ('interface', 'vni-4098'), - ('device', 'n9k'), - ('packet_count', 40), - ('packet_drop_count', 3), - ('packet_drop_percentage', 6.0) - ]) - ], - 'direction-reverse': [ - OrderedDict([ - ('interface', 'vni-4098'), - ('device', 'n9k'), - ('packet_count', 77) - ]), - OrderedDict([ - ('interface', 'vxlan_tunnel1'), - ('device', 'vpp'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/3'), - ('device', 'vpp'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/1'), - ('device', 'vpp'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/0'), - ('device', 'vpp'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'VirtualEthernet0/0/2'), - ('device', 'vpp'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'vxlan_tunnel0'), - ('device', 'vpp'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]), - OrderedDict([ - ('interface', 'vni-4097'), - ('device', 'n9k'), - ('packet_count', 77), - ('packet_drop_count', 0), - ('packet_drop_percentage', 0.0) - ]) - ]} - } - assert result == expected_result -""" - - -# ========================================================================= # Traffic client tests # ========================================================================= @@ -473,7 +297,7 @@ def test_parse_rate_str(): except Exception: return True else: - assert False + return False assert should_raise_error('101') assert should_raise_error('201%') @@ -500,6 +324,38 @@ def test_rate_conversion(): assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888) assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222) +# pps at 10Gbps line rate for 64 byte frames +LR_64B_PPS = 14880952 +LR_1518B_PPS = 812743 + +def assert_equivalence(reference, value, allowance_pct=1): + '''Asserts if a value is equivalent to a reference value with given margin + + :param float reference: reference value to compare to + :param float value: value to compare to reference + :param float allowance_pct: max allowed percentage of margin + 0 : requires exact match + 1 : must be equal within 1% of the reference value + ... + 100: always true + ''' + if reference == 0: + assert value == 0 + else: + assert abs(value - reference) * 100 / reference <= allowance_pct + +def test_load_from_rate(): + assert traffic_utils.get_load_from_rate('100%') == 100 + assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_64B_PPS) + 'pps')) + assert_equivalence(50, traffic_utils.get_load_from_rate(str(LR_64B_PPS / 2) + 'pps')) + assert_equivalence(100, traffic_utils.get_load_from_rate('10Gbps')) + assert_equivalence(50, traffic_utils.get_load_from_rate('5000Mbps')) + assert_equivalence(1, traffic_utils.get_load_from_rate('100Mbps')) + assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS) + 'pps', + avg_frame_size=1518)) + assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS * 2) + 'pps', + avg_frame_size=1518, + line_rate='20Gbps')) """ @pytest.fixture @@ -513,112 +369,14 @@ def traffic_client(monkeypatch): 'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}] } - self.config = AttrDict({ - 'generator_config': { - 'intf_speed': 10000000000 - }, - 'ndr_run': True, - 'pdr_run': True, - 'single_run': False, - 'attempts': 1, - 'measurement': { - 'NDR': 0.0, - 'PDR': 0.1, - 'load_epsilon': 0.1 - } - }) - - self.runner = AttrDict({ - 'time_elapsed': lambda: 30, - 'stop': lambda: None, - 'client': AttrDict({'get_stats': lambda: None}) - }) - - self.current_load = None - self.dummy_stats = { - 50.0: 72.6433562831, - 25.0: 45.6095059858, - 12.5: 0.0, - 18.75: 27.218642979, - 15.625: 12.68585861, - 14.0625: 2.47154392563, - 13.28125: 0.000663797066801, - 12.890625: 0.0, - 13.0859375: 0.0, - 13.18359375: 0.00359387347122, - 13.671875: 0.307939922531, - 13.4765625: 0.0207718516156, - 13.57421875: 0.0661795060969 - } - def mock_modify_load(self, load): self.run_config['rates'][0] = {'rate_percent': str(load)} self.current_load = load - def mock_run_traffic(self): - yield { - 'overall': { - 'drop_rate_percent': self.dummy_stats[self.current_load], - 'rx': { - 'total_pkts': 1, - 'avg_delay_usec': 0.0, - 'max_delay_usec': 0.0, - 'min_delay_usec': 0.0 - } - } - } - monkeypatch.setattr(TrafficClient, '__init__', mock_init) monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load) - monkeypatch.setattr(TrafficClient, 'run_traffic', mock_run_traffic) return TrafficClient() - - -def test_ndr_pdr_search(traffic_client): - expected_results = { - 'pdr': { - 'l2frame_size': '64', - 'initial_rate_type': 'rate_percent', - 'stats': { - 'overall': { - 'drop_rate_percent': 0.0661795060969, - 'min_delay_usec': 0.0, - 'avg_delay_usec': 0.0, - 'max_delay_usec': 0.0 - } - }, - 'load_percent_per_direction': 13.57421875, - 'rate_percent': 13.57422547, - 'rate_bps': 1357422547.0, - 'rate_pps': 2019974.0282738095, - 'duration_sec': 30 - }, - 'ndr': { - 'l2frame_size': '64', - 'initial_rate_type': 'rate_percent', - 'stats': { - 'overall': { - 'drop_rate_percent': 0.0, - 'min_delay_usec': 0.0, - 'avg_delay_usec': 0.0, - 'max_delay_usec': 0.0 - } - }, - 'load_percent_per_direction': 13.0859375, - 'rate_percent': 13.08594422, - 'rate_bps': 1308594422.0, - 'rate_pps': 1947313.1279761905, - 'duration_sec': 30 - } - } - - results = traffic_client.get_ndr_and_pdr() - assert len(results) == 2 - for result in results.values(): - result.pop('timestamp_sec') - result.pop('time_taken_sec') - assert results == expected_results """ @@ -631,7 +389,6 @@ def test_ndr_pdr_search(traffic_client): def setup_module(module): nfvbench.log.setup(mute_stdout=True) - def test_no_credentials(): cred = Credentials('/completely/wrong/path/openrc', None, False) if cred.rc_auth_url: @@ -667,7 +424,8 @@ except ImportError: # pylint: disable=wrong-import-position,ungrouped-imports from nfvbench.traffic_client import Device from nfvbench.traffic_client import IpBlock - +from nfvbench.traffic_client import TrafficClient +from nfvbench.traffic_client import TrafficGeneratorFactory # pylint: enable=wrong-import-position,ungrouped-imports @@ -828,3 +586,95 @@ def test_fluentd(): raise Exception("test") except Exception: logger.exception("got exception") + +def assert_ndr_pdr(stats, ndr, ndr_dr, pdr, pdr_dr): + assert stats['ndr']['rate_percent'] == ndr + assert stats['ndr']['stats']['overall']['drop_percentage'] == ndr_dr + assert_equivalence(pdr, stats['pdr']['rate_percent']) + assert_equivalence(pdr_dr, stats['pdr']['stats']['overall']['drop_percentage']) + +def get_traffic_client(): + config = AttrDict({ + 'traffic_generator': {'host_name': 'nfvbench_tg', + 'default_profile': 'dummy', + 'generator_profile': [{'name': 'dummy', + 'tool': 'dummy', + 'ip': '127.0.0.1', + 'intf_speed': '10Gbps', + 'interfaces': [{'port': 0, 'pci': 0}, + {'port': 1, 'pci': 0}]}], + 'ip_addrs_step': '0.0.0.1', + 'ip_addrs': ['10.0.0.0/8', '20.0.0.0/8'], + 'tg_gateway_ip_addrs': ['1.1.0.100', '2.2.0.100'], + 'tg_gateway_ip_addrs_step': '0.0.0.1', + 'gateway_ip_addrs': ['1.1.0.2', '2.2.0.2'], + 'gateway_ip_addrs_step': '0.0.0.1', + 'udp_src_port': None, + 'udp_dst_port': None}, + 'generator_profile': 'dummy', + 'service_chain': 'PVP', + 'service_chain_count': 1, + 'flow_count': 10, + 'vlan_tagging': True, + 'no_arp': False, + 'duration_sec': 1, + 'interval_sec': 1, + 'single_run': False, + 'ndr_run': True, + 'pdr_run': True, + 'rate': 'ndr_pdr', + 'check_traffic_time_sec': 200, + 'generic_poll_sec': 2, + 'measurement': {'NDR': 0.001, 'PDR': 0.1, 'load_epsilon': 0.1}, + }) + generator_factory = TrafficGeneratorFactory(config) + config.generator_config = generator_factory.get_generator_config(config.generator_profile) + traffic_client = TrafficClient(config, skip_sleep=True) + traffic_client.start_traffic_generator() + traffic_client.set_traffic('64', True) + return traffic_client + +def test_ndr_at_lr(): + traffic_client = get_traffic_client() + tg = traffic_client.gen + + # this is a perfect sut with no loss at LR + tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100) + # tx packets should be line rate for 64B and no drops... + assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0) + # NDR and PDR should be at 100% + traffic_client.ensure_end_to_end() + results = traffic_client.get_ndr_and_pdr() + + assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0) + +def test_ndr_at_50(): + traffic_client = get_traffic_client() + tg = traffic_client.gen + # this is a sut with an NDR of 50% and linear drop rate after NDR up to 20% drops at LR + # (meaning that if you send 100% TX, you will only receive 80% RX) + # the tg requested TX/actual TX ratio is 1up to 50%, after 50% + # is linear up 80% actuak TX when requesting 100% + tg.set_response_curve(lr_dr=20, ndr=50, max_actual_tx=80, max_11_tx=50) + # tx packets should be half line rate for 64B and no drops... + assert tg.get_tx_pps_dropped_pps(50) == (LR_64B_PPS / 2, 0) + # at 100% TX requested, actual TX is 80% where the drop rate is 3/5 of 20% of the actual TX + assert tg.get_tx_pps_dropped_pps(100) == (int(LR_64B_PPS * 0.8), + int(LR_64B_PPS * 0.8 * 0.6 * 0.2)) + results = traffic_client.get_ndr_and_pdr() + assert_ndr_pdr(results, 100.0, 0.0, 100.781, 0.09374) + +def test_ndr_pdr_low_cpu(): + traffic_client = get_traffic_client() + tg = traffic_client.gen + # This test is for the case where the TG is underpowered and cannot send fast enough for the NDR + # true NDR=40%, actual TX at 50% = 30%, actual measured DR is 0% + # The ndr/pdr should bail out with a warning and a best effort measured NDR of 30% + tg.set_response_curve(lr_dr=50, ndr=40, max_actual_tx=60, max_11_tx=0) + # tx packets should be 30% at requested half line rate for 64B and no drops... + assert tg.get_tx_pps_dropped_pps(50) == (int(LR_64B_PPS * 0.3), 0) + results = traffic_client.get_ndr_and_pdr() + assert results + # import pprint + # pp = pprint.PrettyPrinter(indent=4) + # pp.pprint(results) @@ -10,7 +10,7 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = py.test -q -s --basetemp={envtmpdir} {posargs} +commands = py.test -q --basetemp={envtmpdir} {posargs} [testenv:pep8] commands = flake8 {toxinidir} |