# Copyright 2015-2016 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """TestCase base class """ from collections import OrderedDict import copy import csv import logging import math import os import re import time import subprocess from conf import settings as S from conf import get_test_param, merge_spec import core.component_factory as component_factory from core.loader import Loader from core.results.results_constants import ResultsConstants from tools import tasks from tools import hugepages from tools import functions from tools import namespace from tools import veth from tools.teststepstools import TestStepsTools CHECK_PREFIX = 'validate_' class TestCase(object): """TestCase base class In this basic form runs RFC2544 throughput test """ def __init__(self, cfg): """Pull out fields from test config :param cfg: A dictionary of string-value pairs describing the test configuration. Both the key and values strings use well-known values. :param results_dir: Where the csv formatted results are written. """ self._testcase_start_time = time.time() self._hugepages_mounted = False self._traffic_ctl = None self._vnf_ctl = None self._vswitch_ctl = None self._collector = None self._loadgen = None self._output_file = None self._tc_results = None self._settings_original = {} self._settings_paths_modified = False self._testcast_run_time = None # initialization of step driven specific members self._step_check = False # by default don't check result for step driven testcases self._step_vnf_list = {} self._step_result = [] self._step_status = None self._testcase_run_time = None # store all GUEST_ specific settings to keep original values before their expansion for key in S.__dict__: if key.startswith('GUEST_'): self._settings_original[key] = S.getValue(key) self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH'))) self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF'))) self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN'))) test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS')) test_params = merge_spec(test_params, tc_test_params) self._update_settings('TEST_PARAMS', test_params) S.check_test_params() # override all redefined GUEST_ values to have them expanded correctly tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) for key in tmp_test_params: if key.startswith('GUEST_'): S.setValue(key, S.getValue(key)) S.getValue('TEST_PARAMS').pop(key) # update global settings functions.settings_update_paths() # set test parameters; CLI options take precedence to testcase settings self._logger = logging.getLogger(__name__) self.name = cfg['Name'] self.desc = cfg.get('Description', 'No description given.') self.test = cfg.get('TestSteps', None) bidirectional = S.getValue('TRAFFIC')['bidir'] if not isinstance(S.getValue('TRAFFIC')['bidir'], str): raise TypeError( 'Bi-dir value must be of type string') bidirectional = bidirectional.title() # Keep things consistent self.deployment = cfg['Deployment'] self._frame_mod = cfg.get('Frame Modification', None) self._tunnel_type = None self._tunnel_operation = None if self.deployment == 'op2p': self._tunnel_operation = cfg['Tunnel Operation'] if 'Tunnel Type' in cfg: self._tunnel_type = cfg['Tunnel Type'] self._tunnel_type = get_test_param('TUNNEL_TYPE', self._tunnel_type) # check if test requires background load and which generator it uses self._load_cfg = cfg.get('Load', None) if self._load_cfg and 'tool' in self._load_cfg: self._loadgen = self._load_cfg['tool'] else: 
# Copyright 2015 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# ############################
# Collector configuration
# ############################

COLLECTOR = 'Pidstat'
COLLECTOR_DIR = os.path.join(ROOT_DIR, 'tools/collectors')

# processes to be monitored by pidstat
PIDSTAT_MONITOR = ['ovs-vswitchd', 'ovsdb-server', 'qemu-system-x86_64', 'vpp']

# options which will be passed to pidstat
PIDSTAT_OPTIONS = '-dur'

# sampling interval used by pidstat to collect statistics
PIDSTAT_SAMPLE_INTERVAL = 1

# prefix of pidstat's log file; separate log file is created
# for each testcase in the directory with results
LOG_FILE_PIDSTAT = 'pidstat'
'Removing content of shared directory...', True) # directory to share files between host and guest os.makedirs(guest_dir) # copy sources into shared dir only if neccessary guest_loopback = set(S.getValue('GUEST_LOOPBACK')) if 'testpmd' in guest_loopback: try: # exclude whole .git/ subdirectory and all o-files; # It is assumed, that the same RTE_TARGET is used in both host # and VMs; This simplification significantly speeds up testpmd # build. If we will need a different RTE_TARGET in VM, # then we have to build whole DPDK from the scratch in VM. # In that case we can copy just DPDK sources (e.g. by excluding # all items obtained by git status -unormal --porcelain). # NOTE: Excluding RTE_TARGET directory won't help on systems, # where DPDK is built for multiple targets (e.g. for gcc & icc) exclude = [] exclude.append(r'--exclude=.git/') exclude.append(r'--exclude=*.o') tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude + [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''), os.path.join(guest_dir, 'DPDK')], self._logger, 'Copying DPDK to shared directory...', True) except subprocess.CalledProcessError: self._logger.error('Unable to copy DPDK to shared directory') raise if 'l2fwd' in guest_loopback: try: tasks.run_task(['rsync', '-a', '-r', '-l', os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'), os.path.join(guest_dir, 'l2fwd')], self._logger, 'Copying l2fwd to shared directory...', True) except subprocess.CalledProcessError: self._logger.error('Unable to copy l2fwd to shared directory') raise def _mount_hugepages(self): """Mount hugepages if usage of DPDK or Qemu is detected """ # hugepages are needed by DPDK and Qemu if not self._hugepages_mounted and \ (self.deployment.count('v') or \ S.getValue('VSWITCH').lower().count('dpdk') or \ self._vswitch_none or \ self.test and 'vnf' in [step[0][0:3] for step in self.test]): hugepages.mount_hugepages() self._hugepages_mounted = True def _umount_hugepages(self): """Umount hugepages if they were mounted before """ if self._hugepages_mounted: hugepages.umount_hugepages() self._hugepages_mounted = False def _check_for_enough_hugepages(self): """Check to make sure enough hugepages are free to satisfy the test environment. """ hugepages_needed = 0 hugepage_size = hugepages.get_hugepage_size() # get hugepage amounts per guest involved in the test for guest in range(self._vnf_ctl.get_vnfs_number()): hugepages_needed += math.ceil((int(S.getValue( 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size) # get hugepage amounts for each socket on dpdk sock0_mem, sock1_mem = 0, 0 if S.getValue('VSWITCH').lower().count('dpdk'): # the import below needs to remain here and not put into the module # imports because of an exception due to settings not yet loaded from vswitches import ovs_dpdk_vhost if ovs_dpdk_vhost.OvsDpdkVhost.old_dpdk_config(): match = re.search( r'-socket-mem\s+(\d+),(\d+)', ''.join(S.getValue('VSWITCHD_DPDK_ARGS'))) if match: sock0_mem, sock1_mem = (int(match.group(1)) * 1024 / hugepage_size, int(match.group(2)) * 1024 / hugepage_size) else: logging.info( 'Could not parse socket memory config in dpdk params.') else: sock0_mem, sock1_mem = ( S.getValue( 'VSWITCHD_DPDK_CONFIG')['dpdk-socket-mem'].split(',')) sock0_mem, sock1_mem = (int(sock0_mem) * 1024 / hugepage_size, int(sock1_mem) * 1024 / hugepage_size) # If hugepages needed, verify the amounts are free if any([hugepages_needed, sock0_mem, sock1_mem]): free_hugepages = hugepages.get_free_hugepages() if hugepages_needed: logging.info('Need %s hugepages free for guests', hugepages_needed) result1 = free_hugepages >= hugepages_needed free_hugepages -= hugepages_needed else: result1 = True if sock0_mem: logging.info('Need %s hugepages free for dpdk socket 0', sock0_mem) result2 = hugepages.get_free_hugepages('0') >= sock0_mem free_hugepages -= sock0_mem else: result2 = True if sock1_mem: logging.info('Need %s hugepages free for dpdk socket 1', sock1_mem) result3 = hugepages.get_free_hugepages('1') >= sock1_mem free_hugepages -= sock1_mem else: result3 = True logging.info('Need a total of {} total hugepages'.format( hugepages_needed + sock1_mem + sock0_mem)) # The only drawback here is sometimes dpdk doesn't release # its hugepages on a test failure. This could cause a test # to fail when dpdk would be OK to start because it will just # use the previously allocated hugepages. result4 = True if free_hugepages >= 0 else False return all([result1, result2, result3, result4]) else: return True @staticmethod def write_result_to_file(results, output): """Write list of dictionaries to a CSV file. Each element on list will create separate row in output file. If output file already exists, data will be appended at the end, otherwise it will be created. :param results: list of dictionaries. :param output: path to output file. """ with open(output, 'a') as csvfile: logging.info("Write results to file: " + output) fieldnames = TestCase._get_unique_keys(results) writer = csv.DictWriter(csvfile, fieldnames) if not csvfile.tell(): # file is now empty writer.writeheader() for result in results: writer.writerow(result) @staticmethod def _get_unique_keys(list_of_dicts): """Gets unique key values as ordered list of strings in given dicts :param list_of_dicts: list of dictionaries. :returns: list of unique keys(strings). """ result = OrderedDict() for item in list_of_dicts: for key in item.keys(): result[key] = '' return list(result.keys()) def _add_flows(self): """Add flows to the vswitch """ vswitch = self._vswitch_ctl.get_vswitch() # TODO BOM 15-08-07 the frame mod code assumes that the # physical ports are ports 1 & 2. The actual numbers # need to be retrived from the vSwitch and the metadata value # updated accordingly. bridge = S.getValue('VSWITCH_BRIDGE_NAME') if self._frame_mod == "vlan": # 0x8100 => VLAN ethertype self._logger.debug(" **** VLAN ***** ") flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'actions': ['push_vlan:0x8100', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'actions': ['push_vlan:0x8100', 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "mpls": # 0x8847 => MPLS unicast ethertype self._logger.debug(" **** MPLS ***** ") flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'actions': ['push_mpls:0x8847', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'actions': ['push_mpls:0x8847', 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "mac": flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'actions': ['mod_dl_src:22:22:22:22:22:22', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'actions': ['mod_dl_src:11:11:11:11:11:11', 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "dscp": # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding' flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'dl_type':'0x0800', 'actions': ['mod_nw_tos:184', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'dl_type':'0x0800', 'actions': ['mod_nw_tos:184', 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "ttl": # 251 and 241 are the highest prime numbers < 255 flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'dl_type':'0x0800', 'actions': ['mod_nw_ttl:251', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'dl_type':'0x0800', 'actions': ['mod_nw_ttl:241', 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "ip_addr": flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'dl_type':'0x0800', 'actions': ['mod_nw_src:10.10.10.10', 'mod_nw_dst:20.20.20.20', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'dl_type':'0x0800', 'actions': ['mod_nw_src:20.20.20.20', 'mod_nw_dst:10.10.10.10', 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "ip_port": # TODO BOM 15-08-27 The traffic generated is assumed # to be UDP (nw_proto 17d) which is the default case but # we will need to pick up the actual traffic params in use. flow = {'table':'2', 'priority':'1000', 'metadata':'2', 'dl_type':'0x0800', 'nw_proto':'17', 'actions': ['mod_tp_src:44444', 'mod_tp_dst:44444', 'goto_table:3']} vswitch.add_flow(bridge, flow) flow = {'table':'2', 'priority':'1000', 'metadata':'1', 'dl_type':'0x0800', 'nw_proto':'17', 'actions': ['mod_tp_src:44444', 'mod_tp_dst:44444', 'goto_table:3']} vswitch.add_flow(bridge, flow) else: pass # # TestSteps realted methods # def step_report_status(self, label, status): """ Log status of test step """ self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED') def step_stop_vnfs(self): """ Stop all VNFs started by TestSteps """ for vnf in self._step_vnf_list: if self._step_vnf_list[vnf]: self._step_vnf_list[vnf].stop() @staticmethod def step_eval_param(param, STEP): # pylint: disable=invalid-name """ Helper function for #STEP macro evaluation """ if isinstance(param, str): # evaluate every #STEP reference inside parameter itself macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param) if macros: for macro in macros: # pylint: disable=eval-used tmp_val = str(eval(macro[1:])) param = param.replace(macro, tmp_val) return param elif isinstance(param, list) or isinstance(param, tuple): tmp_list = [] for item in param: tmp_list.append(TestCase.step_eval_param(item, STEP)) return tmp_list elif isinstance(param, dict): tmp_dict = {} for (key, value) in param.items(): tmp_dict[key] = TestCase.step_eval_param(value, STEP) return tmp_dict else: return param @staticmethod def step_eval_params(params, step_result): """ Evaluates referrences to results from previous steps """ eval_params = [] # evaluate all parameters if needed for param in params: eval_params.append(TestCase.step_eval_param(param, step_result)) return eval_params def step_run(self): """ Execute actions specified by TestSteps list :return: False if any error was detected True otherwise """ # anything to do? if not self.test: return True # required for VNFs initialization loader = Loader() # initialize list with results self._step_result = [None] * len(self.test) # run test step by step... for i, step in enumerate(self.test): step_ok = not self._step_check if step[0] == 'vswitch': test_object = self._vswitch_ctl.get_vswitch() elif step[0] == 'namespace': test_object = namespace elif step[0] == 'veth': test_object = veth elif step[0] == 'settings': test_object = S elif step[0] == 'tools': test_object = TestStepsTools() step[1] = step[1].title() elif step[0] == 'trafficgen': test_object = self._traffic_ctl # in case of send_traffic or send_traffic_async methods, ensure # that specified traffic values are merged with existing self._traffic if step[1].startswith('send_traffic'): tmp_traffic = copy.deepcopy(self._traffic) tmp_traffic.update(step[2]) step[2] = tmp_traffic elif step[0].startswith('vnf'): if not self._step_vnf_list[step[0]]: # initialize new VM self._step_vnf_list[step[0]] = loader.get_vnf_class()() test_object = self._step_vnf_list[step[0]] elif step[0] == 'wait': input(os.linesep + "Step {}: Press Enter to continue with " "the next step...".format(i) + os.linesep + os.linesep) continue else: self._logger.error("Unsupported test object %s", step[0]) self._step_status = {'status' : False, 'details' : ' '.join(step)} self.step_report_status("Step '{}'".format(' '.join(step)), self._step_status['status']) return False test_method = getattr(test_object, step[1]) if self._step_check: test_method_check = getattr(test_object, CHECK_PREFIX + step[1]) else: test_method_check = None step_params = [] try: # eval parameters, but use only valid step_results # to support negative indexes step_params = TestCase.step_eval_params(step[2:], self._step_result[:i]) step_log = '{} {}'.format(' '.join(step[:2]), step_params) self._logger.debug("Step %s '%s' start", i, step_log) self._step_result[i] = test_method(*step_params) self._logger.debug("Step %s '%s' results '%s'", i, step_log, self._step_result[i]) time.sleep(5) if self._step_check: step_ok = test_method_check(self._step_result[i], *step_params) except (AssertionError, AttributeError, IndexError) as ex: step_ok = False self._logger.error("Step %s raised %s", i, type(ex).__name__) if self._step_check: self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok) if not step_ok: self._step_status = {'status' : False, 'details' : step_log} # Stop all VNFs started by TestSteps self.step_stop_vnfs() return False # all steps processed without any issue return True