aboutsummaryrefslogtreecommitdiffstats
path: root/docs/testing/user
AgeCommit message (Expand)AuthorFilesLines
2018-11-09Merge "[docs] Resolve "document isn't included in any toctree" warning" into ...Rex Lee2-1/+7
2018-11-09[docs] Resolve "nested numbering in toctree?" warningEmma Foley1-1/+0
2018-11-09[docs] Resolve "document isn't included in any toctree" warningopnfv-7.0.0Emma Foley2-1/+7
2018-11-09[docs][userguide] Update formatting for Influx instructions in ch04Emma Foley1-15/+15
2018-11-09[userguide] Update the directory structure listingEmma Foley1-12/+13
2018-11-09[docs] Fix "Error in xxx directive" warningEmma Foley3-5/+2
2018-11-09[docs] Fix docs {un,}indent warnings in Testcase definitionsEmma Foley16-106/+148
2018-11-09Merge "[docs] Remove vtc results chapter and references" into stable/gambiaRex Lee1-11/+0
2018-11-09[docs] Update header levelsEmma Foley4-49/+70
2018-11-09[docs] Remove vtc results chapter and referencesEmma Foley1-11/+0
2018-11-09[docs] Fix unresolved reference warningEmma Foley1-2/+2
2018-11-09[docs] Update broken and redirected linksEmma Foley8-21/+21
2018-11-09[docs] Fix links to remove references to submodulesAric Gardner1-4/+4
2018-11-09Merge "[docs] Update mailing list to use #yardstick" into stable/gambiaRex Lee1-1/+1
2018-11-06Merge "[docs] Add vEPC test case preparation steps" into stable/gambiaAbhijit Sinha2-3/+157
2018-11-06[docs] Update mailing list to use #yardstickEmma Foley1-1/+1
2018-11-06Merge "[docs] Test case documentation for vEPC testing" into stable/gambiaRex Lee6-0/+810
2018-11-06Merge changes I526c328f,I7bb14112,Idfe77900 into stable/gambiaRex Lee2-0/+113
2018-11-05[docs] Add vEPC test case preparation stepsOrest Voznyy2-3/+157
2018-11-01[docs] Test case documentation for vEPC testingAbhijit Sinha6-0/+810
2018-10-31Merge "[docs] Update the glossary" into stable/gambiaEmma Foley1-13/+62
2018-10-30[docs] Update the glossaryEmma Foley1-13/+62
2018-10-30[docs] Update on Yardstick installationStepan Andrushko2-0/+113
2018-10-30Adds option parameters to TC074Ameed Ashour1-0/+11
2018-08-29Updated documentation for stanalone contextStepan Andrushko1-15/+85
2018-08-16Merge "Disk Initialization and Custom Workload"Rex Lee1-19/+53
2018-08-10Disk Initialization and Custom Workloadmbeierl1-19/+53
2018-07-23[docs] Remove the VTC chapter in the userguideEmma Foley7-139/+1
2018-06-27Corrected scale-up command line argumentsChornyi, TarasX1-1/+1
2018-06-21Merge "Add scale out TCs with availability zone support"Rodolfo Alonso Hernandez1-1/+111
2018-06-18Update pmd/lcore mask for OVS-DPDK contextStepan Andrushko1-0/+36
2018-05-25Add scale out TCs with availability zone supportMytnyk, Volodymyr1-1/+111
2018-05-23Test case spec for SDN controller resilience in HA configGeorg Kunz2-0/+197
2018-05-20Merge "Test case spec for SDN Virtual Switch resilience."Emma Foley2-0/+185
2018-05-16Merge "Test case description and configuration file for yardstick_tc088"Jing Lu1-0/+129
2018-05-16Merge "Test case description and configuration file for yardstick_tc089"Jing Lu1-0/+129
2018-05-07Test case spec for SDN Virtual Switch resilience.Dimitrios Markou2-0/+185
2018-05-04Add IxNetwork API Python Binding packageRodolfo Alonso Hernandez2-75/+41
2018-04-23Merge "[docs] Updated baremetal instalation instructions"Rodolfo Alonso Hernandez1-0/+9
2018-04-13Bugfix: interface down HA tc050 attack and monitor logistic wrongrexlee87761-27/+25
2018-04-06[docs] Updated baremetal instalation instructionsChornyi, TarasX1-0/+9
2018-04-03[userguide] Use literalinclude directive in 05-operationEmma Foley1-75/+9
2018-04-03[docs] update the user guide for "yardstick-operation"Emma Foley3-0/+366
2018-04-03[docs] Move user guide chapters to make room for new chapter.Emma Foley14-551/+400
2018-03-24Merge "Test case spec for SDN controller resilience in non-HA config"Ross Brattain2-0/+183
2018-03-23Merge "Test case description and configuration file for yardstick_tc090:Contr...Ross Brattain1-0/+151
2018-03-22Added scale-up documentationChornyi, TarasX1-9/+45
2018-03-22Merge "Test case description and configuration file for yardstick_tc091:Contr...Rex Lee1-0/+138
2018-03-19Bugfix: sutie -> suite in documentationchenjiankun1-1/+1
2018-03-16NSB userguide: run TC in heat context & sriov portsMytnyk, Volodymyr5-4/+486
pan class="p">), 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'), 'final' : 'test_report.rst', 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst') } _LOGGER = logging.getLogger() def parse_arguments(): """ Parse command line arguments. """ class _SplitTestParamsAction(argparse.Action): """ Parse and split the '--test-params' argument. This expects either 'x=y', 'x=y,z' or 'x' (implicit true) values. For multiple overrides use a ; separated list for e.g. --test-params 'x=z; y=(a,b)' """ def __call__(self, parser, namespace, values, option_string=None): results = {} for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values): param = param.strip() value = value.strip() if len(param): if len(value): results[param] = value else: results[param] = True setattr(namespace, self.dest, results) class _ValidateFileAction(argparse.Action): """Validate a file can be read from before using it. """ def __call__(self, parser, namespace, values, option_string=None): if not os.path.isfile(values): raise argparse.ArgumentTypeError( 'the path \'%s\' is not a valid path' % values) elif not os.access(values, os.R_OK): raise argparse.ArgumentTypeError( 'the path \'%s\' is not accessible' % values) setattr(namespace, self.dest, values) class _ValidateDirAction(argparse.Action): """Validate a directory can be written to before using it. """ def __call__(self, parser, namespace, values, option_string=None): if not os.path.isdir(values): raise argparse.ArgumentTypeError( 'the path \'%s\' is not a valid path' % values) elif not os.access(values, os.W_OK): raise argparse.ArgumentTypeError( 'the path \'%s\' is not accessible' % values) setattr(namespace, self.dest, values) def list_logging_levels(): """Give a summary of all available logging levels. :return: List of verbosity level names in decreasing order of verbosity """ return sorted(VERBOSITY_LEVELS.keys(), key=lambda x: VERBOSITY_LEVELS[x]) parser = argparse.ArgumentParser(prog=__file__, formatter_class= argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--version', action='version', version='%(prog)s 0.2') parser.add_argument('--list', '--list-tests', action='store_true', help='list all tests and exit') parser.add_argument('--list-trafficgens', action='store_true', help='list all traffic generators and exit') parser.add_argument('--list-collectors', action='store_true', help='list all system metrics loggers and exit') parser.add_argument('--list-vswitches', action='store_true', help='list all system vswitches and exit') parser.add_argument('--list-fwdapps', action='store_true', help='list all system forwarding applications and exit') parser.add_argument('--list-vnfs', action='store_true', help='list all system vnfs and exit') parser.add_argument('--list-settings', action='store_true', help='list effective settings configuration and exit') parser.add_argument('exact_test_name', nargs='*', help='Exact names of\ tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\ runs only the two tests with those exact names.\ To run all tests omit both positional args and --tests arg.') group = parser.add_argument_group('test selection options') group.add_argument('-m', '--mode', help='vsperf mode of operation;\ Values: "normal" - execute vSwitch, VNF and traffic generator;\ "trafficgen" - execute only traffic generator; "trafficgen-off" \ - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \ and VNF but pause before traffic transmission ', default='normal') group.add_argument('-f', '--test-spec', help='test specification file') group.add_argument('-d', '--test-dir', help='directory containing tests') group.add_argument('-t', '--tests', help='Comma-separated list of terms \ indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\ name contains RFC2544 less those containing "p2p"; "!back2back" - \ run all tests except those containing back2back') group.add_argument('--verbosity', choices=list_logging_levels(), help='debug level') group.add_argument('--integration', action='store_true', help='execute integration tests') group.add_argument('--trafficgen', help='traffic generator to use') group.add_argument('--vswitch', help='vswitch implementation to use') group.add_argument('--fwdapp', help='packet forwarding application to use') group.add_argument('--vnf', help='vnf to use') group.add_argument('--sysmetrics', help='system metrics logger to use') group = parser.add_argument_group('test behavior options') group.add_argument('--xunit', action='store_true', help='enable xUnit-formatted output') group.add_argument('--xunit-dir', action=_ValidateDirAction, help='output directory of xUnit-formatted output') group.add_argument('--load-env', action='store_true', help='enable loading of settings from the environment') group.add_argument('--conf-file', action=_ValidateFileAction, help='settings file') group.add_argument('--test-params', action=_SplitTestParamsAction, help='csv list of test parameters: key=val; e.g. ' 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; ' 'GUEST_LOOPBACK=["l2fwd"] ...') group.add_argument('--opnfvpod', help='name of POD in opnfv') args = vars(parser.parse_args()) return args def configure_logging(level): """Configure logging. """ log_file_default = os.path.join( settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT')) log_file_host_cmds = os.path.join( settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS')) log_file_traffic_gen = os.path.join( settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_TRAFFIC_GEN')) _LOGGER.setLevel(logging.DEBUG) stream_logger = logging.StreamHandler(sys.stdout) stream_logger.setLevel(VERBOSITY_LEVELS[level]) stream_logger.setFormatter(logging.Formatter( '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s')) _LOGGER.addHandler(stream_logger) file_logger = logging.FileHandler(filename=log_file_default) file_logger.setLevel(logging.DEBUG) _LOGGER.addHandler(file_logger) class CommandFilter(logging.Filter): """Filter out strings beginning with 'cmd :'""" def filter(self, record): return record.getMessage().startswith(tasks.CMD_PREFIX) class TrafficGenCommandFilter(logging.Filter): """Filter out strings beginning with 'gencmd :'""" def filter(self, record): return record.getMessage().startswith(trafficgen.CMD_PREFIX) cmd_logger = logging.FileHandler(filename=log_file_host_cmds) cmd_logger.setLevel(logging.DEBUG) cmd_logger.addFilter(CommandFilter()) _LOGGER.addHandler(cmd_logger) gen_logger = logging.FileHandler(filename=log_file_traffic_gen) gen_logger.setLevel(logging.DEBUG) gen_logger.addFilter(TrafficGenCommandFilter()) _LOGGER.addHandler(gen_logger) def apply_filter(tests, tc_filter): """Allow a subset of tests to be conveniently selected :param tests: The list of Tests from which to select. :param tc_filter: A case-insensitive string of comma-separated terms indicating the Tests to select. e.g. 'RFC' - select all tests whose name contains 'RFC' e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or 'burst' e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC' or 'burst' and from these remove any containing 'p2p'. e.g. '' - empty string selects all tests. :return: A list of the selected Tests. """ # if negative filter is first we have to start with full list of tests if tc_filter.strip()[0] == '!': result = tests else: result = [] if tc_filter is None: tc_filter = "" for term in [x.strip() for x in tc_filter.lower().split(",")]: if not term or term[0] != '!': # Add matching tests from 'tests' into results result.extend([test for test in tests \ if test['Name'].lower().find(term) >= 0]) else: # Term begins with '!' so we remove matching tests result = [test for test in result \ if test['Name'].lower().find(term[1:]) < 0] return result def check_and_set_locale(): """ Function will check locale settings. In case, that it isn't configured properly, then default values specified by DEFAULT_LOCALE will be used. """ system_locale = locale.getdefaultlocale() if None in system_locale: os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE') _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s", system_locale, locale.getdefaultlocale()) def generate_final_report(): """ Function will check if partial test results are available and generates final report in rst format. """ path = settings.getValue('RESULTS_PATH') # check if there are any results in rst format rst_results = glob.glob(os.path.join(path, 'result*rst')) if len(rst_results): try: test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final'])) # create report caption directly - it is not worth to execute jinja machinery if settings.getValue('VSWITCH').lower() != 'none': pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0] else: pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0] report_caption = '{}\n{} {}\n{}\n\n'.format( '============================================================', 'Performance report for', pkt_processor, '============================================================') with open(_TEMPLATE_RST['tmp'], 'w') as file_: file_.write(report_caption) retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'], ' '.join(rst_results), _TEMPLATE_RST['foot'], test_report), shell=True) if retval == 0 and os.path.isfile(test_report): _LOGGER.info('Overall test report written to "%s"', test_report) else: _LOGGER.error('Generatrion of overall test report has failed.') # remove temporary file os.remove(_TEMPLATE_RST['tmp']) except subprocess.CalledProcessError: _LOGGER.error('Generatrion of overall test report has failed.') def enable_sriov(nic_list): """ Enable SRIOV for given enhanced PCI IDs :param nic_list: A list of enhanced PCI IDs """ # detect if sriov is required sriov_nic = {} for nic in nic_list: if networkcard.is_sriov_nic(nic): tmp_nic = nic.split('|') if tmp_nic[0] in sriov_nic: if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]: sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:]) else: sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])}) # sriov is required for some NICs if len(sriov_nic): for nic in sriov_nic: # check if SRIOV is supported and enough virt interfaces are available if not networkcard.is_sriov_supported(nic) \ or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]: # if not, enable and set appropriate number of VFs if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1): _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic) raise else: _LOGGER.debug("SRIOV enabled for NIC %s", nic) # WORKAROUND: it has been observed with IXGBE(VF) driver, # that NIC doesn't correclty dispatch traffic to VFs based # on their MAC address. Unbind and bind to the same driver # solves this issue. networkcard.reinit_vfs(nic) # After SRIOV is enabled it takes some time until network drivers # properly initialize all cards. # Wait also in case, that SRIOV was already configured as it can be # configured automatically just before vsperf execution. time.sleep(2) return True return False def disable_sriov(nic_list): """ Disable SRIOV for given PCI IDs :param nic_list: A list of enhanced PCI IDs """ for nic in nic_list: if networkcard.is_sriov_nic(nic): if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0): _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic) raise else: _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0]) def handle_list_options(args): """ Process --list cli arguments if needed :param args: A dictionary with all CLI arguments """ if args['list_trafficgens']: print(Loader().get_trafficgens_printable()) sys.exit(0) if args['list_collectors']: print(Loader().get_collectors_printable()) sys.exit(0) if args['list_vswitches']: print(Loader().get_vswitches_printable()) sys.exit(0) if args['list_vnfs']: print(Loader().get_vnfs_printable()) sys.exit(0) if args['list_fwdapps']: print(Loader().get_pktfwds_printable()) sys.exit(0) if args['list_settings']: print(str(settings)) sys.exit(0) if args['list']: # configure tests if args['integration']: testcases = settings.getValue('INTEGRATION_TESTS') else: testcases = settings.getValue('PERFORMANCE_TESTS') print("Available Tests:") print("================") for test in testcases: print('* %-30s %s' % ('%s:' % test['Name'], test['Description'])) sys.exit(0) def vsperf_finalize(): """ Clean up before exit """ # remove directory if no result files were created try: results_path = settings.getValue('RESULTS_PATH') if os.path.exists(results_path): files_list = os.listdir(results_path) if files_list == []: _LOGGER.info("Removing empty result directory: " + results_path) shutil.rmtree(results_path) except AttributeError: # skip it if parameter doesn't exist pass # disable SRIOV if needed try: if settings.getValue('SRIOV_ENABLED'): disable_sriov(settings.getValue('WHITELIST_NICS_ORIG')) except AttributeError: # skip it if parameter doesn't exist pass class MockTestCase(unittest.TestCase): """Allow use of xmlrunner to generate Jenkins compatible output without using xmlrunner to actually run tests. Usage: suite = unittest.TestSuite() suite.addTest(MockTestCase('Test1 passed ', True, 'Test1')) suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2')) xmlrunner.XMLTestRunner(...).run(suite) """ def __init__(self, msg, is_pass, test_name): #remember the things self.msg = msg self.is_pass = is_pass #dynamically create a test method with the right name #but point the method at our generic test method setattr(MockTestCase, test_name, self.generic_test) super(MockTestCase, self).__init__(test_name) def generic_test(self): """Provide a generic function that raises or not based on how self.is_pass was set in the constructor""" self.assertTrue(self.is_pass, self.msg) def main(): """Main function. """ args = parse_arguments() # configure settings settings.load_from_dir(os.path.join(_CURR_DIR, 'conf')) # Load non performance/integration tests if args['integration']: settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration')) # load command line parameters first in case there are settings files # to be used settings.load_from_dict(args) if args['conf_file']: settings.load_from_file(args['conf_file']) if args['load_env']: settings.load_from_env() # reload command line parameters since these should take higher priority # than both a settings file and environment variables settings.load_from_dict(args) # set dpdk and ovs paths accorfing to VNF and VSWITCH functions.settings_update_paths() # if required, handle list-* operations handle_list_options(args) configure_logging(settings.getValue('VERBOSITY')) # check and fix locale check_and_set_locale() # configure trafficgens if args['trafficgen']: trafficgens = Loader().get_trafficgens() if args['trafficgen'] not in trafficgens: _LOGGER.error('There are no trafficgens matching \'%s\' found in' ' \'%s\'. Exiting...', args['trafficgen'], settings.getValue('TRAFFICGEN_DIR')) sys.exit(1) # configuration validity checks if args['vswitch']: vswitch_none = 'none' == args['vswitch'].strip().lower() if vswitch_none: settings.setValue('VSWITCH', 'none') else: vswitches = Loader().get_vswitches() if args['vswitch'] not in vswitches: _LOGGER.error('There are no vswitches matching \'%s\' found in' ' \'%s\'. Exiting...', args['vswitch'], settings.getValue('VSWITCH_DIR')) sys.exit(1) if args['fwdapp']: settings.setValue('PKTFWD', args['fwdapp']) fwdapps = Loader().get_pktfwds() if args['fwdapp'] not in fwdapps: _LOGGER.error('There are no forwarding application' ' matching \'%s\' found in' ' \'%s\'. Exiting...', args['fwdapp'], settings.getValue('PKTFWD_DIR')) sys.exit(1) if args['vnf']: vnfs = Loader().get_vnfs() if args['vnf'] not in vnfs: _LOGGER.error('there are no vnfs matching \'%s\' found in' ' \'%s\'. exiting...', args['vnf'], settings.getValue('VNF_DIR')) sys.exit(1) if args['exact_test_name'] and args['tests']: _LOGGER.error("Cannot specify tests with both positional args and --test.") sys.exit(1) # modify NIC configuration to decode enhanced PCI IDs wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS')) settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig) # sriov handling is performed on checked/expanded PCI IDs settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig)) nic_list = [] for nic in wl_nics_orig: tmp_nic = networkcard.get_nic_info(nic) if tmp_nic: nic_list.append({'pci' : tmp_nic, 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf', 'mac' : networkcard.get_mac(tmp_nic), 'driver' : networkcard.get_driver(tmp_nic), 'device' : networkcard.get_device_name(tmp_nic)}) else: _LOGGER.error("Invalid network card PCI ID: '%s'", nic) vsperf_finalize() raise settings.setValue('NICS', nic_list) # for backward compatibility settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list)) settings.setValue('mode', args['mode']) # generate results directory name date = datetime.datetime.fromtimestamp(time.time()) results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S') results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir) settings.setValue('RESULTS_PATH', results_path) # create results directory if not os.path.exists(results_path): _LOGGER.info("Creating result directory: " + results_path) os.makedirs(results_path) if settings.getValue('mode') == 'trafficgen': # execute only traffic generator _LOGGER.debug("Executing traffic generator:") loader = Loader() # set traffic details, so they can be passed to traffic ctl traffic = copy.deepcopy(TRAFFIC_DEFAULTS) traffic.update({'traffic_type': get_test_param('traffic_type', TRAFFIC_DEFAULTS['traffic_type']), 'bidir': get_test_param('bidirectional', TRAFFIC_DEFAULTS['bidir']), 'multistream': int(get_test_param('multistream', TRAFFIC_DEFAULTS['multistream'])), 'stream_type': get_test_param('stream_type', TRAFFIC_DEFAULTS['stream_type']), 'frame_rate': int(get_test_param('iload', TRAFFIC_DEFAULTS['frame_rate']))}) traffic_ctl = component_factory.create_traffic( traffic['traffic_type'], loader.get_trafficgen_class()) with traffic_ctl: traffic_ctl.send_traffic(traffic) _LOGGER.debug("Traffic Results:") traffic_ctl.print_results() # write results into CSV file result_file = os.path.join(results_path, "result.csv") PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file) else: # configure tests if args['integration']: testcases = settings.getValue('INTEGRATION_TESTS') else: testcases = settings.getValue('PERFORMANCE_TESTS') if args['exact_test_name']: exact_names = args['exact_test_name'] # positional args => exact matches only selected_tests = [test for test in testcases if test['Name'] in exact_names] elif args['tests']: # --tests => apply filter to select requested tests selected_tests = apply_filter(testcases, args['tests']) else: # Default - run all tests selected_tests = testcases if not len(selected_tests): _LOGGER.error("No tests matched --tests option or positional args. Done.") vsperf_finalize() sys.exit(1) # run tests suite = unittest.TestSuite() for cfg in selected_tests: test_name = cfg.get('Name', '<Name not set>') try: if args['integration']: test = IntegrationTestCase(cfg) else: test = PerformanceTestCase(cfg) test.run() suite.addTest(MockTestCase('', True, test.name)) #pylint: disable=broad-except except (Exception) as ex: _LOGGER.exception("Failed to run test: %s", test_name) suite.addTest(MockTestCase(str(ex), False, test_name)) _LOGGER.info("Continuing with next test...") # generate final rst report with results of all executed TCs generate_final_report() if settings.getValue('XUNIT'): xmlrunner.XMLTestRunner( output=settings.getValue('XUNIT_DIR'), outsuffix="", verbosity=0).run(suite) if args['opnfvpod']: pod_name = args['opnfvpod'] installer_name = settings.getValue('OPNFV_INSTALLER') opnfv_url = settings.getValue('OPNFV_URL') pkg_list = settings.getValue('PACKAGE_LIST') int_data = {'vanilla': False, 'pod': pod_name, 'installer': installer_name, 'pkg_list': pkg_list, 'db_url': opnfv_url} if settings.getValue('VSWITCH').endswith('Vanilla'): int_data['vanilla'] = True opnfvdashboard.results2opnfv_dashboard(results_path, int_data) # cleanup before exit vsperf_finalize() if __name__ == "__main__": main()