From 95f2491ed89ac99b0d8bd006b4a13cbeb1eb96ce Mon Sep 17 00:00:00 2001 From: "Michael S. Pedersen" Date: Tue, 3 Dec 2019 11:38:32 +0000 Subject: NFVBENCH-153 Add support for python3 JIRA: NFVBENCH-153 Done using 2to3-3.6 with additional changes to fix data parsing and testing (tox) Signed-off-by: Michael S. Pedersen Change-Id: I242902f800da543d780507828c9bd1fbf409da6d --- nfvbench/cfg.default.yaml | 2 +- nfvbench/chain_router.py | 5 +-- nfvbench/chain_runner.py | 10 ++--- nfvbench/chain_workers.py | 2 - nfvbench/chaining.py | 17 ++++---- nfvbench/cleanup.py | 16 ++++---- nfvbench/compute.py | 4 +- nfvbench/config.py | 4 +- nfvbench/config_plugin.py | 9 +---- nfvbench/credentials.py | 2 +- nfvbench/factory.py | 4 +- nfvbench/fluentd.py | 4 +- nfvbench/nfvbench.py | 42 +++++++++---------- nfvbench/nfvbenchd.py | 51 +++++++++++------------ nfvbench/packet_stats.py | 8 ++-- nfvbench/stats_manager.py | 6 +-- nfvbench/summarizer.py | 34 ++++++++-------- nfvbench/traffic_client.py | 76 +++++++++++++++++------------------ nfvbench/traffic_gen/dummy.py | 8 ++-- nfvbench/traffic_gen/traffic_base.py | 13 ++---- nfvbench/traffic_gen/traffic_utils.py | 22 +++++----- nfvbench/traffic_gen/trex_gen.py | 36 ++++++++--------- nfvbench/traffic_server.py | 6 +-- nfvbench/utils.py | 6 +-- 24 files changed, 180 insertions(+), 207 deletions(-) (limited to 'nfvbench') diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml index 551b1c4..d3a3fb9 100755 --- a/nfvbench/cfg.default.yaml +++ b/nfvbench/cfg.default.yaml @@ -310,7 +310,7 @@ cores: # https://trex-tgn.cisco.com/trex/doc/trex_stateless.html#_tutorial_field_engine_significantly_improve_performance # If cache_size = 0 (or empty): no cache will be used by TRex (default) # If cache_size < 0: cache_size will be set to flow count value -cache_size: +cache_size: 0 # The cache size is actually limited by the number of 64B mbufs configured in the trex platform configuration (see Trex manual 6.2.2. Memory section configuration) # Trex will use 1 x 64B mbuf per pre-built cached packet, assuming 1 pre-built cached packet per flow, it means for very large number of flows, the number of configured mbuf_64 will need to be set accordingly. mbuf_64: diff --git a/nfvbench/chain_router.py b/nfvbench/chain_router.py index 9372716..ac89476 100644 --- a/nfvbench/chain_router.py +++ b/nfvbench/chain_router.py @@ -39,15 +39,12 @@ import time from netaddr import IPAddress from netaddr import IPNetwork -from log import LOG +from .log import LOG class ChainException(Exception): """Exception while operating the chains.""" - pass - - class ChainRouter(object): """Could be a shared router across all chains or a chain private router.""" diff --git a/nfvbench/chain_runner.py b/nfvbench/chain_runner.py index 833373c..418d667 100644 --- a/nfvbench/chain_runner.py +++ b/nfvbench/chain_runner.py @@ -23,11 +23,11 @@ The ChainRunner class is in charge of coordinating: from collections import OrderedDict -from chaining import ChainManager -from log import LOG -from specs import ChainType -from stats_manager import StatsManager -from traffic_client import TrafficClient +from .chaining import ChainManager +from .log import LOG +from .specs import ChainType +from .stats_manager import StatsManager +from .traffic_client import TrafficClient class ChainRunner(object): diff --git a/nfvbench/chain_workers.py b/nfvbench/chain_workers.py index 0ed2648..e332d7b 100644 --- a/nfvbench/chain_workers.py +++ b/nfvbench/chain_workers.py @@ -43,7 +43,6 @@ class BasicWorker(object): Specialized workers can insert their own interface stats inside each existing packet path stats for every chain. """ - pass def update_interface_stats(self, diff=False): """Update all interface stats. @@ -53,4 +52,3 @@ class BasicWorker(object): Make sure that the interface stats inserted in insert_interface_stats() are updated with proper values """ - pass diff --git a/nfvbench/chaining.py b/nfvbench/chaining.py index d035a40..a8d6295 100644 --- a/nfvbench/chaining.py +++ b/nfvbench/chaining.py @@ -54,10 +54,10 @@ from neutronclient.neutron import client as neutronclient from novaclient.client import Client from attrdict import AttrDict -from chain_router import ChainRouter -import compute -from log import LOG -from specs import ChainType +from .chain_router import ChainRouter +from . import compute +from .log import LOG +from .specs import ChainType # Left and right index for network and port lists LEFT = 0 RIGHT = 1 @@ -77,9 +77,6 @@ BOOT_SCRIPT_PATHNAME = os.path.join(os.path.dirname(os.path.abspath(__file__)), class ChainException(Exception): """Exception while operating the chains.""" - pass - - class NetworkEncaps(object): """Network encapsulation.""" @@ -706,8 +703,8 @@ class ChainVnf(object): # here we MUST wait until this instance is resolved otherwise subsequent # VNF creation can be placed in other hypervisors! config = self.manager.config - max_retries = (config.check_traffic_time_sec + - config.generic_poll_sec - 1) / config.generic_poll_sec + max_retries = int((config.check_traffic_time_sec + + config.generic_poll_sec - 1) / config.generic_poll_sec) retry = 0 for retry in range(max_retries): status = self.get_status() @@ -1380,7 +1377,7 @@ class ChainManager(object): return: the hypervisor where the matching port runs or None if not found """ # _existing_ports is a dict of list of ports indexed by network id - for port_list in self.get_existing_ports().values(): + for port_list in list(self.get_existing_ports().values()): for port in port_list: try: if port['mac_address'] == mac: diff --git a/nfvbench/cleanup.py b/nfvbench/cleanup.py index 6a79a63..22ed3c1 100644 --- a/nfvbench/cleanup.py +++ b/nfvbench/cleanup.py @@ -22,8 +22,8 @@ from novaclient.client import Client from novaclient.exceptions import NotFound from tabulate import tabulate -import credentials as credentials -from log import LOG +from . import credentials +from .log import LOG class ComputeCleaner(object): @@ -300,20 +300,20 @@ class Cleaner(object): LOG.info("NFVbench will delete resources shown...") clean_options = None if prompt: - answer = raw_input("Do you want to delete all ressources? (y/n) ") + answer = input("Do you want to delete all ressources? (y/n) ") if answer.lower() != 'y': - print "What kind of resources do you want to delete?" + print("What kind of resources do you want to delete?") all_option = "" all_option_codes = [] for cleaner in self.cleaners: code = cleaner.get_cleaner_code() - print "%s: %s" % (code[0], code) + print(("%s: %s" % (code[0], code))) all_option += code[0] all_option_codes.append(code) - print "a: all resources - a shortcut for '%s'" % all_option + print(("a: all resources - a shortcut for '%s'" % all_option)) all_option_codes.append("all resources") - print "q: quit" - answer_res = raw_input(":").lower() + print("q: quit") + answer_res = input(":").lower() # Check only first character because answer_res can be "flavor" and it is != all if answer_res[0] == "a": clean_options = all_option diff --git a/nfvbench/compute.py b/nfvbench/compute.py index 84e3774..f6f179d 100644 --- a/nfvbench/compute.py +++ b/nfvbench/compute.py @@ -24,7 +24,7 @@ except ImportError: import keystoneauth1 import novaclient -from log import LOG +from .log import LOG class Compute(object): @@ -50,7 +50,7 @@ class Compute(object): retry = 0 try: # check image is file/url based. - with open(image_file) as f_image: + with open(image_file, 'rb') as f_image: img = self.glance_client.images.create(name=str(final_image_name), disk_format="qcow2", container_format="bare", diff --git a/nfvbench/config.py b/nfvbench/config.py index dba0962..4cc9c86 100644 --- a/nfvbench/config.py +++ b/nfvbench/config.py @@ -16,7 +16,7 @@ from attrdict import AttrDict import yaml -from log import LOG +from .log import LOG def config_load(file_name, from_cfg=None, whitelist_keys=None): """Load a yaml file into a config dict, merge with from_cfg if not None @@ -64,7 +64,7 @@ def config_loads(cfg_text, from_cfg=None, whitelist_keys=None): def _validate_config(subset, superset, whitelist_keys): def get_err_config(subset, superset): result = {} - for k, v in subset.items(): + for k, v in list(subset.items()): if k not in whitelist_keys: if k not in superset: result.update({k: v}) diff --git a/nfvbench/config_plugin.py b/nfvbench/config_plugin.py index a6759cd..0596fcf 100644 --- a/nfvbench/config_plugin.py +++ b/nfvbench/config_plugin.py @@ -18,19 +18,15 @@ This module is used to override the configuration with platform specific constraints and extensions """ import abc -import specs +from . import specs -class ConfigPluginBase(object): +class ConfigPluginBase(object, metaclass=abc.ABCMeta): """Base class for config plugins.""" - __metaclass__ = abc.ABCMeta - class InitializationFailure(Exception): """Used in case of any init failure.""" - pass - def __init__(self, config): """Save configuration.""" if not config: @@ -97,7 +93,6 @@ class ConfigPlugin(ConfigPluginBase): def validate_config(self, config, openstack_spec): """Nothing to validate by default.""" - pass def prepare_results_config(self, cfg): """Nothing to add the results by default.""" diff --git a/nfvbench/credentials.py b/nfvbench/credentials.py index 9956c6f..d9a67e6 100644 --- a/nfvbench/credentials.py +++ b/nfvbench/credentials.py @@ -21,7 +21,7 @@ import getpass from keystoneauth1.identity import v2 from keystoneauth1.identity import v3 from keystoneauth1 import session -from log import LOG +from .log import LOG class Credentials(object): diff --git a/nfvbench/factory.py b/nfvbench/factory.py index cad5a43..0d4b042 100644 --- a/nfvbench/factory.py +++ b/nfvbench/factory.py @@ -15,8 +15,8 @@ # """Factory for creating worker and config plugin instances.""" -import chain_workers as workers -from config_plugin import ConfigPlugin +from . import chain_workers as workers +from .config_plugin import ConfigPlugin class BasicFactory(object): diff --git a/nfvbench/fluentd.py b/nfvbench/fluentd.py index ad0ea34..535d640 100644 --- a/nfvbench/fluentd.py +++ b/nfvbench/fluentd.py @@ -114,7 +114,7 @@ class FluentLogHandler(logging.Handler): def __get_highest_level(self): if self.__error_counter > 0: return logging.ERROR - elif self.__warning_counter > 0: + if self.__warning_counter > 0: return logging.WARNING return logging.INFO @@ -122,7 +122,7 @@ class FluentLogHandler(logging.Handler): highest_level = self.__get_highest_level() if highest_level == logging.INFO: return "GOOD RUN" - elif highest_level == logging.WARNING: + if highest_level == logging.WARNING: return "RUN WITH WARNINGS" return "RUN WITH ERRORS" diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py index 18a7d2b..eb86dea 100644 --- a/nfvbench/nfvbench.py +++ b/nfvbench/nfvbench.py @@ -27,20 +27,20 @@ from attrdict import AttrDict import pbr.version from pkg_resources import resource_string -from __init__ import __version__ -from chain_runner import ChainRunner -from cleanup import Cleaner -from config import config_load -from config import config_loads -import credentials as credentials -from fluentd import FluentLogHandler -import log -from log import LOG -from nfvbenchd import WebServer -from specs import ChainType -from specs import Specs -from summarizer import NFVBenchSummarizer -import utils +from .__init__ import __version__ +from .chain_runner import ChainRunner +from .cleanup import Cleaner +from .config import config_load +from .config import config_loads +from . import credentials +from .fluentd import FluentLogHandler +from . import log +from .log import LOG +from .nfvbenchd import WebServer +from .specs import ChainType +from .specs import Specs +from .summarizer import NFVBenchSummarizer +from . import utils fluent_logger = None @@ -87,7 +87,7 @@ class NFVBench(object): try: # recalc the running config based on the base config and options for this run self._update_config(opts) - if self.config.cache_size < 0: + if int(self.config.cache_size) < 0: self.config.cache_size = self.config.flow_count # check that an empty openrc file (no OpenStack) is only allowed # with EXT chain @@ -538,7 +538,7 @@ def main(): log.set_level(debug=opts.debug) if opts.version: - print pbr.version.VersionInfo('nfvbench').version_string_with_vcs() + print((pbr.version.VersionInfo('nfvbench').version_string_with_vcs())) sys.exit(0) if opts.summary: @@ -546,12 +546,12 @@ def main(): result = json.load(json_data) if opts.user_label: result['config']['user_label'] = opts.user_label - print NFVBenchSummarizer(result, fluent_logger) + print((NFVBenchSummarizer(result, fluent_logger))) sys.exit(0) # show default config in text/yaml format if opts.show_default_config: - print default_cfg + print((default_cfg.decode("utf-8"))) sys.exit(0) config.name = '' @@ -633,7 +633,7 @@ def main(): # show running config in json format if opts.show_config: - print json.dumps(config, sort_keys=True, indent=4) + print((json.dumps(config, sort_keys=True, indent=4))) sys.exit(0) # update the config in the config plugin as it might have changed @@ -670,7 +670,7 @@ def main(): raise Exception(err_msg) # remove unfilled values - opts = {k: v for k, v in vars(opts).iteritems() if v is not None} + opts = {k: v for k, v in list(vars(opts).items()) if v is not None} # get CLI args params = ' '.join(str(e) for e in sys.argv[1:]) result = nfvbench_instance.run(opts, params) @@ -686,7 +686,7 @@ def main(): 'status': NFVBench.STATUS_ERROR, 'error_message': traceback.format_exc() }) - print str(exc) + print((str(exc))) finally: if fluent_logger: # only send a summary record if there was an actual nfvbench run or diff --git a/nfvbench/nfvbenchd.py b/nfvbench/nfvbenchd.py index ae89e7a..73e1342 100644 --- a/nfvbench/nfvbenchd.py +++ b/nfvbench/nfvbenchd.py @@ -15,7 +15,7 @@ # import json -import Queue +import queue from threading import Thread import uuid @@ -23,13 +23,13 @@ from flask import Flask from flask import jsonify from flask import request -from summarizer import NFVBenchSummarizer +from .summarizer import NFVBenchSummarizer -from log import LOG -from utils import byteify -from utils import RunLock +from .log import LOG +from .utils import byteify +from .utils import RunLock -from __init__ import __version__ +from .__init__ import __version__ STATUS_OK = 'OK' STATUS_ERROR = 'ERROR' @@ -57,7 +57,7 @@ def get_uuid(): class Ctx(object): MAXLEN = 5 - run_queue = Queue.Queue() + run_queue = queue.Queue() busy = False result = None results = {} @@ -101,16 +101,15 @@ class Ctx(object): res = Ctx.results[request_id] except KeyError: return None - + # pylint: disable=unsubscriptable-object if Ctx.result and request_id == Ctx.result['request_id']: Ctx.result = None - - return res - else: - res = Ctx.result - if res: - Ctx.result = None return res + # pylint: enable=unsubscriptable-object + res = Ctx.result + if res: + Ctx.result = None + return res @staticmethod def is_busy(): @@ -159,20 +158,18 @@ def setup_flask(): return jsonify(res) # result for given request_id not found return jsonify(result_json(STATUS_NOT_FOUND, not_found_msg, request_id)) - else: - if Ctx.is_busy(): - # task still pending, return with request_id - return jsonify(result_json(STATUS_PENDING, - pending_msg, - Ctx.get_current_request_id())) - - res = Ctx.get_result() - if res: - return jsonify(res) - return jsonify(not_busy_json) + if Ctx.is_busy(): + # task still pending, return with request_id + return jsonify(result_json(STATUS_PENDING, + pending_msg, + Ctx.get_current_request_id())) - return app + res = Ctx.get_result() + if res: + return jsonify(res) + return jsonify(not_busy_json) + return app class WebServer(object): """This class takes care of the web server. Caller should simply create an instance @@ -200,7 +197,7 @@ class WebServer(object): # print config try: # remove unfilled values as we do not want them to override default values with None - config = {k: v for k, v in config.items() if v is not None} + config = {k: v for k, v in list(config.items()) if v is not None} with RunLock(): if self.fluent_logger: self.fluent_logger.start_new_run() diff --git a/nfvbench/packet_stats.py b/nfvbench/packet_stats.py index 3203b72..63b1f11 100644 --- a/nfvbench/packet_stats.py +++ b/nfvbench/packet_stats.py @@ -21,7 +21,7 @@ PacketPathStatsManager manages all packet path stats for all chains. import copy -from traffic_gen.traffic_base import Latency +from .traffic_gen.traffic_base import Latency class InterfaceStats(object): """A class to hold the RX and TX counters for a virtual or physical interface. @@ -292,7 +292,7 @@ class PacketPathStatsManager(object): chains['total'] = agg_pps.get_stats(reverse) for index, pps in enumerate(self.pps_list): - chains[index] = pps.get_stats(reverse) + chains[str(index)] = pps.get_stats(reverse) return {'interfaces': self._get_if_agg_name(reverse), 'chains': chains} @@ -307,11 +307,11 @@ class PacketPathStatsManager(object): 'Forward': { 'interfaces': ['Port0', 'vhost0', 'Port1'], 'chains': { - 0: {'packets': [2000054, 1999996, 1999996], + '0': {'packets': [2000054, 1999996, 1999996], 'min_usec': 10, 'max_usec': 187, 'avg_usec': 45}, - 1: {...}, + '1': {...}, 'total': {...} } }, diff --git a/nfvbench/stats_manager.py b/nfvbench/stats_manager.py index 98ac413..5ba3a1a 100644 --- a/nfvbench/stats_manager.py +++ b/nfvbench/stats_manager.py @@ -15,9 +15,9 @@ # import time -from log import LOG -from packet_stats import PacketPathStatsManager -from stats_collector import IntervalCollector +from .log import LOG +from .packet_stats import PacketPathStatsManager +from .stats_collector import IntervalCollector class StatsManager(object): diff --git a/nfvbench/summarizer.py b/nfvbench/summarizer.py index 7520076..de62a73 100644 --- a/nfvbench/summarizer.py +++ b/nfvbench/summarizer.py @@ -47,7 +47,7 @@ def _annotate_chain_stats(chain_stats, nodrop_marker='=>'): In the case of shared net, some columns in packets array can have ''. Some columns cab also be None which means the data is not available. """ - for stats in chain_stats.values(): + for stats in list(chain_stats.values()): packets = stats['packets'] count = len(packets) if count > 1: @@ -97,7 +97,7 @@ class Formatter(object): def standard(data): if isinstance(data, int): return Formatter.int(data) - elif isinstance(data, float): + if isinstance(data, float): return Formatter.float(4)(data) return Formatter.fixed(data) @@ -130,7 +130,7 @@ class Formatter(object): def percentage(data): if data is None: return '' - elif math.isnan(data): + if math.isnan(data): return '-' return Formatter.suffix('%')(Formatter.float(4)(data)) @@ -139,7 +139,7 @@ class Table(object): """ASCII readable table class.""" def __init__(self, header): - header_row, self.formatters = zip(*header) + header_row, self.formatters = list(zip(*header)) self.data = [header_row] self.columns = len(header_row) @@ -195,7 +195,7 @@ class Summarizer(object): def _put_dict(self, data): with self._create_block(False): - for key, value in data.iteritems(): + for key, value in list(data.items()): if isinstance(value, dict): self._put(key + ':') self._put_dict(value) @@ -297,7 +297,7 @@ class NFVBenchSummarizer(Summarizer): if network_benchmark['versions']: self._put('Versions:') with self._create_block(): - for component, version in network_benchmark['versions'].iteritems(): + for component, version in list(network_benchmark['versions'].items()): self._put(component + ':', version) if self.config['ndr_run'] or self.config['pdr_run']: @@ -308,7 +308,7 @@ class NFVBenchSummarizer(Summarizer): if self.config['pdr_run']: self._put('PDR:', self.config['measurement']['PDR']) self._put('Service chain:') - for result in network_benchmark['service_chain'].iteritems(): + for result in list(network_benchmark['service_chain'].items()): with self._create_block(): self.__chain_summarize(*result) @@ -325,13 +325,13 @@ class NFVBenchSummarizer(Summarizer): self._put('Bidirectional:', traffic_benchmark['bidirectional']) self._put('Flow count:', traffic_benchmark['flow_count']) self._put('Service chains count:', traffic_benchmark['service_chain_count']) - self._put('Compute nodes:', traffic_benchmark['compute_nodes'].keys()) + self._put('Compute nodes:', list(traffic_benchmark['compute_nodes'].keys())) self.__record_header_put('profile', traffic_benchmark['profile']) self.__record_header_put('bidirectional', traffic_benchmark['bidirectional']) self.__record_header_put('flow_count', traffic_benchmark['flow_count']) self.__record_header_put('sc_count', traffic_benchmark['service_chain_count']) - self.__record_header_put('compute_nodes', traffic_benchmark['compute_nodes'].keys()) + self.__record_header_put('compute_nodes', list(traffic_benchmark['compute_nodes'].keys())) with self._create_block(False): self._put() if not self.config['no_traffic']: @@ -345,7 +345,7 @@ class NFVBenchSummarizer(Summarizer): except KeyError: pass - for entry in traffic_benchmark['result'].iteritems(): + for entry in list(traffic_benchmark['result'].items()): if 'warning' in entry: continue self.__chain_analysis_summarize(*entry) @@ -391,7 +391,7 @@ class NFVBenchSummarizer(Summarizer): summary_table = Table(self.ndr_pdr_header) if self.config['ndr_run']: - for frame_size, analysis in traffic_result.iteritems(): + for frame_size, analysis in list(traffic_result.items()): if frame_size == 'warning': continue summary_table.add_row([ @@ -414,7 +414,7 @@ class NFVBenchSummarizer(Summarizer): 'max_delay_usec': analysis['ndr']['stats']['overall']['max_delay_usec'] }}) if self.config['pdr_run']: - for frame_size, analysis in traffic_result.iteritems(): + for frame_size, analysis in list(traffic_result.items()): if frame_size == 'warning': continue summary_table.add_row([ @@ -437,7 +437,7 @@ class NFVBenchSummarizer(Summarizer): 'max_delay_usec': analysis['pdr']['stats']['overall']['max_delay_usec'] }}) if self.config['single_run']: - for frame_size, analysis in traffic_result.iteritems(): + for frame_size, analysis in list(traffic_result.items()): summary_table.add_row([ frame_size, analysis['stats']['overall']['drop_rate_percent'], @@ -485,11 +485,11 @@ class NFVBenchSummarizer(Summarizer): chain_stats: { 'interfaces': ['Port0', 'drop %'', 'vhost0', 'Port1'], 'chains': { - 0: {'packets': [2000054, '-0.023%', 1999996, 1999996], + '0': {'packets': [2000054, '-0.023%', 1999996, 1999996], 'lat_min_usec': 10, 'lat_max_usec': 187, 'lat_avg_usec': 45}, - 1: {...}, + '1': {...}, 'total': {...} } } @@ -503,13 +503,13 @@ class NFVBenchSummarizer(Summarizer): lat_map = {'lat_avg_usec': 'Avg lat.', 'lat_min_usec': 'Min lat.', 'lat_max_usec': 'Max lat.'} - if 'lat_avg_usec' in chains[0]: + if 'lat_avg_usec' in chains['0']: lat_keys = ['lat_avg_usec', 'lat_min_usec', 'lat_max_usec'] for key in lat_keys: header.append((lat_map[key], Formatter.standard)) table = Table(header) - for chain in sorted(chains.keys()): + for chain in sorted(list(chains.keys()), key=str): row = [chain] + chains[chain]['packets'] for lat_key in lat_keys: row.append('{:,} usec'.format(chains[chain][lat_key])) diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py index 6d870f6..a8573b0 100755 --- a/nfvbench/traffic_client.py +++ b/nfvbench/traffic_client.py @@ -28,21 +28,18 @@ from trex.stl.api import STLError from trex.stl.api import UDP # pylint: enable=import-error -from log import LOG -from packet_stats import InterfaceStats -from packet_stats import PacketPathStats -from stats_collector import IntervalCollector -from stats_collector import IterationCollector -import traffic_gen.traffic_utils as utils -from utils import cast_integer +from .log import LOG +from .packet_stats import InterfaceStats +from .packet_stats import PacketPathStats +from .stats_collector import IntervalCollector +from .stats_collector import IterationCollector +from .traffic_gen import traffic_utils as utils +from .utils import cast_integer class TrafficClientException(Exception): """Generic traffic client exception.""" - pass - - class TrafficRunner(object): """Serialize various steps required to run traffic.""" @@ -204,7 +201,7 @@ class Device(object): - VM macs discovered using openstack API - dest MACs provisioned in config file """ - self.vtep_dst_mac = map(str, dest_macs) + self.vtep_dst_mac = list(map(str, dest_macs)) def set_dest_macs(self, dest_macs): """Set the list of dest MACs indexed by the chain id. @@ -213,7 +210,7 @@ class Device(object): - VM macs discovered using openstack API - dest MACs provisioned in config file """ - self.dest_macs = map(str, dest_macs) + self.dest_macs = list(map(str, dest_macs)) def get_dest_macs(self): """Get the list of dest macs for this device. @@ -269,14 +266,14 @@ class Device(object): # calculated as (total_flows + chain_count - 1) / chain_count # - the first chain will have the remainder # example 11 flows and 3 chains => 3, 4, 4 - flows_per_chain = (self.flow_count + self.chain_count - 1) / self.chain_count - cur_chain_flow_count = self.flow_count - flows_per_chain * (self.chain_count - 1) + flows_per_chain = int((self.flow_count + self.chain_count - 1) / self.chain_count) + cur_chain_flow_count = int(self.flow_count - flows_per_chain * (self.chain_count - 1)) peer = self.get_peer_device() self.ip_block.reset_reservation() peer.ip_block.reset_reservation() dest_macs = self.get_dest_macs() - for chain_idx in xrange(self.chain_count): + for chain_idx in range(self.chain_count): src_ip_first, src_ip_last = self.ip_block.reserve_ip_range(cur_chain_flow_count) dst_ip_first, dst_ip_last = peer.ip_block.reserve_ip_range(cur_chain_flow_count) @@ -317,7 +314,7 @@ class Device(object): @staticmethod def int_to_ip(nvalue): """Convert an IP address from numeric to string.""" - return socket.inet_ntoa(struct.pack("!I", nvalue)) + return socket.inet_ntoa(struct.pack("!I", int(nvalue))) class GeneratorConfig(object): @@ -420,7 +417,7 @@ class GeneratorConfig(object): raise TrafficClientException('Dest MAC list %s must have %d entries' % (dest_macs, self.config.service_chain_count)) self.devices[port_index].set_vtep_dst_mac(dest_macs) - LOG.info('Port %d: vtep dst MAC %s', port_index, set([str(mac) for mac in dest_macs])) + LOG.info('Port %d: vtep dst MAC %s', port_index, {str(mac) for mac in dest_macs}) def get_dest_macs(self): """Return the list of dest macs indexed by port.""" @@ -512,10 +509,10 @@ class TrafficClient(object): def _get_generator(self): tool = self.tool.lower() if tool == 'trex': - from traffic_gen import trex_gen + from .traffic_gen import trex_gen return trex_gen.TRex(self) if tool == 'dummy': - from traffic_gen import dummy + from .traffic_gen import dummy return dummy.DummyTG(self) raise TrafficClientException('Unsupported generator tool name:' + self.tool) @@ -533,7 +530,7 @@ class TrafficClient(object): if len(matching_profiles) > 1: raise TrafficClientException('Multiple traffic profiles with name: ' + traffic_profile_name) - elif not matching_profiles: + if not matching_profiles: raise TrafficClientException('Cannot find traffic profile: ' + traffic_profile_name) return matching_profiles[0].l2frame_size @@ -593,8 +590,8 @@ class TrafficClient(object): self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False, e2e=True) # ensures enough traffic is coming back - retry_count = (self.config.check_traffic_time_sec + - self.config.generic_poll_sec - 1) / self.config.generic_poll_sec + retry_count = int((self.config.check_traffic_time_sec + + self.config.generic_poll_sec - 1) / self.config.generic_poll_sec) # we expect to see packets coming from 2 unique MAC per chain # because there can be flooding in the case of shared net @@ -612,7 +609,7 @@ class TrafficClient(object): get_mac_id = lambda packet: packet['binary'][56:62] else: get_mac_id = lambda packet: packet['binary'][6:12] - for it in xrange(retry_count): + for it in range(retry_count): self.gen.clear_stats() self.gen.start_traffic() self.gen.start_capture() @@ -625,7 +622,7 @@ class TrafficClient(object): self.gen.fetch_capture_packets() self.gen.stop_capture() for packet in self.gen.packet_list: - mac_id = get_mac_id(packet) + mac_id = get_mac_id(packet).decode('latin-1') src_mac = ':'.join(["%02x" % ord(x) for x in mac_id]) if src_mac in mac_map and self.is_udp(packet): port, chain = mac_map[src_mac] @@ -741,29 +738,29 @@ class TrafficClient(object): """Collect final stats for previous run.""" stats = self.gen.get_stats() retDict = {'total_tx_rate': stats['total_tx_rate']} - for port in self.PORTS: - retDict[port] = {'tx': {}, 'rx': {}} tx_keys = ['total_pkts', 'total_pkt_bytes', 'pkt_rate', 'pkt_bit_rate'] rx_keys = tx_keys + ['dropped_pkts'] for port in self.PORTS: + port_stats = {'tx': {}, 'rx': {}} for key in tx_keys: - retDict[port]['tx'][key] = int(stats[port]['tx'][key]) + port_stats['tx'][key] = int(stats[port]['tx'][key]) for key in rx_keys: try: - retDict[port]['rx'][key] = int(stats[port]['rx'][key]) + port_stats['rx'][key] = int(stats[port]['rx'][key]) except ValueError: - retDict[port]['rx'][key] = 0 - retDict[port]['rx']['avg_delay_usec'] = cast_integer( + port_stats['rx'][key] = 0 + port_stats['rx']['avg_delay_usec'] = cast_integer( stats[port]['rx']['avg_delay_usec']) - retDict[port]['rx']['min_delay_usec'] = cast_integer( + port_stats['rx']['min_delay_usec'] = cast_integer( stats[port]['rx']['min_delay_usec']) - retDict[port]['rx']['max_delay_usec'] = cast_integer( + port_stats['rx']['max_delay_usec'] = cast_integer( stats[port]['rx']['max_delay_usec']) - retDict[port]['drop_rate_percent'] = self.__get_dropped_rate(retDict[port]) + port_stats['drop_rate_percent'] = self.__get_dropped_rate(port_stats) + retDict[str(port)] = port_stats - ports = sorted(retDict.keys()) + ports = sorted(list(retDict.keys()), key=str) if self.run_config['bidirectional']: retDict['overall'] = {'tx': {}, 'rx': {}} for key in tx_keys: @@ -804,6 +801,7 @@ class TrafficClient(object): def __format_output_stats(self, stats): for key in self.PORTS + ['overall']: + key = str(key) interface = stats[key] stats[key] = { 'tx_pkts': interface['tx']['total_pkts'], @@ -818,7 +816,7 @@ class TrafficClient(object): return stats def __targets_found(self, rate, targets, results): - for tag, target in targets.iteritems(): + for tag, target in list(targets.items()): LOG.info('Found %s (%s) load: %s', tag, target, rate) self.__ndr_pdr_found(tag, rate) results[tag]['timestamp_sec'] = time.time() @@ -854,7 +852,7 @@ class TrafficClient(object): # Split target dicts based on the avg drop rate left_targets = {} right_targets = {} - for tag, target in targets.iteritems(): + for tag, target in list(targets.items()): if stats['overall']['drop_rate_percent'] <= target: # record the best possible rate found for this target results[tag] = rates @@ -987,8 +985,8 @@ class TrafficClient(object): # because we want each direction to have the far end RX rates, # use the far end index (1-idx) to retrieve the RX rates for idx, key in enumerate(["direction-forward", "direction-reverse"]): - tx_rate = results["stats"][idx]["tx"]["total_pkts"] / self.config.duration_sec - rx_rate = results["stats"][1 - idx]["rx"]["total_pkts"] / self.config.duration_sec + tx_rate = results["stats"][str(idx)]["tx"]["total_pkts"] / self.config.duration_sec + rx_rate = results["stats"][str(1 - idx)]["rx"]["total_pkts"] / self.config.duration_sec r[key] = { "orig": self.__convert_rates(self.run_config['rates'][idx]), "tx": self.__convert_rates({'rate_pps': tx_rate}), @@ -999,7 +997,7 @@ class TrafficClient(object): for direction in ['orig', 'tx', 'rx']: total[direction] = {} for unit in ['rate_percent', 'rate_bps', 'rate_pps']: - total[direction][unit] = sum([float(x[direction][unit]) for x in r.values()]) + total[direction][unit] = sum([float(x[direction][unit]) for x in list(r.values())]) r['direction-total'] = total return r diff --git a/nfvbench/traffic_gen/dummy.py b/nfvbench/traffic_gen/dummy.py index ef7f272..7fd3fdb 100644 --- a/nfvbench/traffic_gen/dummy.py +++ b/nfvbench/traffic_gen/dummy.py @@ -13,8 +13,8 @@ # under the License. from nfvbench.log import LOG -from traffic_base import AbstractTrafficGenerator -import traffic_utils as utils +from .traffic_base import AbstractTrafficGenerator +from . import traffic_utils as utils class DummyTG(AbstractTrafficGenerator): @@ -176,8 +176,8 @@ class DummyTG(AbstractTrafficGenerator): def fetch_capture_packets(self): def _get_packet_capture(mac): # convert text to binary - src_mac = mac.replace(':', '').decode('hex') - return {'binary': 'SSSSSS' + src_mac} + src_mac = bytearray.fromhex(mac.replace(':', '')).decode() + return {'binary': bytes('SSSSSS' + src_mac, 'ascii')} # for packet capture, generate 2*scc random packets # normally we should generate packets coming from the right dest macs diff --git a/nfvbench/traffic_gen/traffic_base.py b/nfvbench/traffic_gen/traffic_base.py index 9c78d7e..3bff7da 100644 --- a/nfvbench/traffic_gen/traffic_base.py +++ b/nfvbench/traffic_gen/traffic_base.py @@ -16,7 +16,7 @@ import abc import sys from nfvbench.log import LOG -import traffic_utils +from . import traffic_utils class Latency(object): @@ -27,7 +27,7 @@ class Latency(object): latency_list: aggregate all latency values from list if not None """ - self.min_usec = sys.maxint + self.min_usec = sys.maxsize self.max_usec = 0 self.avg_usec = 0 self.hdrh = None @@ -42,15 +42,12 @@ class Latency(object): def available(self): """Return True if latency information is available.""" - return self.min_usec != sys.maxint + return self.min_usec != sys.maxsize class TrafficGeneratorException(Exception): """Exception for traffic generator.""" - pass - - class AbstractTrafficGenerator(object): def __init__(self, traffic_client): @@ -106,7 +103,6 @@ class AbstractTrafficGenerator(object): def clear_streamblock(self): """Clear all streams from the traffic generator.""" - pass @abc.abstractmethod def resolve_arp(self): @@ -116,7 +112,6 @@ class AbstractTrafficGenerator(object): else a dict of list of dest macs indexed by port# the dest macs in the list are indexed by the chain id """ - pass @abc.abstractmethod def get_macs(self): @@ -124,7 +119,6 @@ class AbstractTrafficGenerator(object): return: a list of MAC addresses indexed by the port# """ - pass @abc.abstractmethod def get_port_speed_gbps(self): @@ -132,4 +126,3 @@ class AbstractTrafficGenerator(object): return: a list of speed in Gbps indexed by the port# """ - pass diff --git a/nfvbench/traffic_gen/traffic_utils.py b/nfvbench/traffic_gen/traffic_utils.py index f856267..c875a5d 100644 --- a/nfvbench/traffic_gen/traffic_utils.py +++ b/nfvbench/traffic_gen/traffic_utils.py @@ -54,12 +54,11 @@ def convert_rates(l2frame_size, rate, intf_speed): pps = bps_to_pps(bps, avg_packet_size) else: raise Exception('Traffic config needs to have a rate type key') - return { 'initial_rate_type': initial_rate_type, - 'rate_pps': int(pps), + 'rate_pps': int(float(pps)), 'rate_percent': load, - 'rate_bps': int(bps) + 'rate_bps': int(float(bps)) } @@ -113,23 +112,22 @@ def parse_rate_str(rate_str): rate_pps = rate_pps[:-1] except KeyError: multiplier = 1 - rate_pps = int(rate_pps.strip()) * multiplier + rate_pps = int(float(rate_pps.strip()) * multiplier) if rate_pps <= 0: raise Exception('%s is out of valid range' % rate_str) return {'rate_pps': str(rate_pps)} - elif rate_str.endswith('ps'): + if rate_str.endswith('ps'): rate = rate_str.replace('ps', '').strip() bit_rate = bitmath.parse_string(rate).bits if bit_rate <= 0: raise Exception('%s is out of valid range' % rate_str) return {'rate_bps': str(int(bit_rate))} - elif rate_str.endswith('%'): + if rate_str.endswith('%'): rate_percent = float(rate_str.replace('%', '').strip()) if rate_percent <= 0 or rate_percent > 100.0: raise Exception('%s is out of valid range (must be 1-100%%)' % rate_str) return {'rate_percent': str(rate_percent)} - else: - raise Exception('Unknown rate string format %s' % rate_str) + raise Exception('Unknown rate string format %s' % rate_str) def get_load_from_rate(rate_str, avg_frame_size=64, line_rate='10Gbps'): '''From any rate string (with unit) return the corresponding load (in % unit) @@ -172,10 +170,10 @@ def to_rate_str(rate): if 'rate_pps' in rate: pps = rate['rate_pps'] return '{}pps'.format(pps) - elif 'rate_bps' in rate: + if 'rate_bps' in rate: bps = rate['rate_bps'] return '{}bps'.format(bps) - elif 'rate_percent' in rate: + if 'rate_percent' in rate: load = rate['rate_percent'] return '{}%'.format(load) assert False @@ -185,7 +183,7 @@ def to_rate_str(rate): def nan_replace(d): """Replaces every occurence of 'N/A' with float nan.""" - for k, v in d.iteritems(): + for k, v in d.items(): if isinstance(v, dict): nan_replace(v) elif v == 'N/A': @@ -200,5 +198,5 @@ def mac_to_int(mac): def int_to_mac(i): """Converts integer representation of MAC address to hex string.""" mac = format(i, 'x').zfill(12) - blocks = [mac[x:x + 2] for x in xrange(0, len(mac), 2)] + blocks = [mac[x:x + 2] for x in range(0, len(mac), 2)] return ':'.join(blocks) diff --git a/nfvbench/traffic_gen/trex_gen.py b/nfvbench/traffic_gen/trex_gen.py index 53786f7..c2e0854 100644 --- a/nfvbench/traffic_gen/trex_gen.py +++ b/nfvbench/traffic_gen/trex_gen.py @@ -25,12 +25,6 @@ from nfvbench.traffic_server import TRexTrafficServer from nfvbench.utils import cast_integer from nfvbench.utils import timeout from nfvbench.utils import TimeoutError -from traffic_base import AbstractTrafficGenerator -from traffic_base import TrafficGeneratorException -import traffic_utils as utils -from traffic_utils import IMIX_AVG_L2_FRAME_SIZE -from traffic_utils import IMIX_L2_SIZES -from traffic_utils import IMIX_RATIOS # pylint: disable=import-error from trex.common.services.trex_service_arp import ServiceARP @@ -57,9 +51,15 @@ from trex.stl.api import ThreeBytesField from trex.stl.api import UDP from trex.stl.api import XByteField - # pylint: enable=import-error +from .traffic_base import AbstractTrafficGenerator +from .traffic_base import TrafficGeneratorException +from . import traffic_utils as utils +from .traffic_utils import IMIX_AVG_L2_FRAME_SIZE +from .traffic_utils import IMIX_L2_SIZES +from .traffic_utils import IMIX_RATIOS + class VXLAN(Packet): """VxLAN class.""" @@ -511,7 +511,7 @@ class TRex(AbstractTrafficGenerator): def __connect_after_start(self): # after start, Trex may take a bit of time to initialize # so we need to retry a few times - for it in xrange(self.config.generic_retry_count): + for it in range(self.config.generic_retry_count): try: time.sleep(1) self.client.connect() @@ -519,7 +519,7 @@ class TRex(AbstractTrafficGenerator): except Exception as ex: if it == (self.config.generic_retry_count - 1): raise - LOG.info("Retrying connection to TRex (%s)...", ex.message) + LOG.info("Retrying connection to TRex (%s)...", ex.msg) def connect(self): """Connect to the TRex server.""" @@ -584,7 +584,7 @@ class TRex(AbstractTrafficGenerator): if os.path.isfile(logpath): # Wait for TRex to finish writing error message last_size = 0 - for _ in xrange(self.config.generic_retry_count): + for _ in range(self.config.generic_retry_count): size = os.path.getsize(logpath) if size == last_size: # probably not writing anymore @@ -609,7 +609,7 @@ class TRex(AbstractTrafficGenerator): LOG.info("Restarting TRex ...") self.__stop_server() # Wait for server stopped - for _ in xrange(self.config.generic_retry_count): + for _ in range(self.config.generic_retry_count): time.sleep(1) if not self.client.is_connected(): LOG.info("TRex is stopped...") @@ -625,7 +625,7 @@ class TRex(AbstractTrafficGenerator): self.client.release(ports=ports) self.client.server_shutdown() except STLError as e: - LOG.warn('Unable to stop TRex. Error: %s', e) + LOG.warning('Unable to stop TRex. Error: %s', e) else: LOG.info('Using remote TRex. Unable to stop TRex') @@ -690,12 +690,12 @@ class TRex(AbstractTrafficGenerator): arp_dest_macs[port] = dst_macs LOG.info('ARP resolved successfully for port %s', port) break - else: - retry = attempt + 1 - LOG.info('Retrying ARP for: %s (retry %d/%d)', - unresolved, retry, self.config.generic_retry_count) - if retry < self.config.generic_retry_count: - time.sleep(self.config.generic_poll_sec) + + retry = attempt + 1 + LOG.info('Retrying ARP for: %s (retry %d/%d)', + unresolved, retry, self.config.generic_retry_count) + if retry < self.config.generic_retry_count: + time.sleep(self.config.generic_poll_sec) else: LOG.error('ARP timed out for port %s (resolved %d out of %d)', port, diff --git a/nfvbench/traffic_server.py b/nfvbench/traffic_server.py index 8d1cf0f..c87f8c3 100644 --- a/nfvbench/traffic_server.py +++ b/nfvbench/traffic_server.py @@ -16,7 +16,7 @@ import os import subprocess import yaml -from log import LOG +from .log import LOG class TrafficServerException(Exception): @@ -71,7 +71,7 @@ class TRexTrafficServer(TrafficServer): try: result = yaml.safe_load(stream) except yaml.YAMLError as exc: - print exc + print(exc) return result def __save_config(self, generator_config, filename): @@ -119,7 +119,7 @@ class TRexTrafficServer(TrafficServer): try: threads = ",".join([repr(thread) for thread in core.threads]) except TypeError: - LOG.warn("No threads defined for socket %s", core.socket) + LOG.warning("No threads defined for socket %s", core.socket) core_result = """ - socket : {socket} threads : [{threads}]""".format(socket=core.socket, threads=threads) diff --git a/nfvbench/utils.py b/nfvbench/utils.py index ecbb55a..3974fd7 100644 --- a/nfvbench/utils.py +++ b/nfvbench/utils.py @@ -23,7 +23,7 @@ import errno import fcntl from functools import wraps import json -from log import LOG +from .log import LOG class TimeoutError(Exception): @@ -74,7 +74,7 @@ def save_json_result(result, json_file, std_json_path, service_chain, service_ch def byteify(data, ignore_dicts=False): # if this is a unicode string, return its string representation - if isinstance(data, unicode): + if isinstance(data, str): return data.encode('utf-8') # if this is a list of values, return list of byteified values if isinstance(data, list): @@ -83,7 +83,7 @@ def byteify(data, ignore_dicts=False): # but only if we haven't already byteified it if isinstance(data, dict) and not ignore_dicts: return {byteify(key, ignore_dicts=ignore_dicts): byteify(value, ignore_dicts=ignore_dicts) - for key, value in data.iteritems()} + for key, value in list(data.items())} # if it's anything else, return it in its original form return data -- cgit 1.2.3-korg