diff options
Diffstat (limited to 'testcases')
| -rw-r--r-- | testcases/__init__.py | 2 | ||||
| -rw-r--r-- | testcases/integration.py | 173 | ||||
| -rw-r--r-- | testcases/performance.py | 38 | ||||
| -rw-r--r-- | testcases/testcase.py | 106 |
4 files changed, 276 insertions, 43 deletions
diff --git a/testcases/__init__.py b/testcases/__init__.py index addf63df..0b6b77e4 100644 --- a/testcases/__init__.py +++ b/testcases/__init__.py @@ -15,3 +15,5 @@ """This module contains test definitions. """ from testcases.testcase import (TestCase) +from testcases.performance import (PerformanceTestCase) +from testcases.integration import (IntegrationTestCase) diff --git a/testcases/integration.py b/testcases/integration.py new file mode 100644 index 00000000..ecaed14f --- /dev/null +++ b/testcases/integration.py @@ -0,0 +1,173 @@ +# Copyright 2015-2016 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""IntegrationTestCase class +""" + +import os +import time +import logging + +from testcases import TestCase +from conf import settings as S +from collections import OrderedDict + +CHECK_PREFIX = 'validate_' + +class IntegrationTestCase(TestCase): + """IntegrationTestCase class + """ + + def __init__(self, cfg, results_dir): + """ Testcase initialization + """ + self._type = 'integration' + super(IntegrationTestCase, self).__init__(cfg, results_dir) + self._logger = logging.getLogger(__name__) + self._inttest = None + + def report_status(self, label, status): + """ Log status of test step + """ + self._logger.debug("%s ... %s", label, 'OK' if status else 'FAILED') + + def run_initialize(self): + """ Prepare test execution environment + """ + super(IntegrationTestCase, self).run_initialize() + self._inttest = {'status' : True, 'details' : ''} + + def run(self): + """Run the test + + All setup and teardown through controllers is included. + """ + def eval_step_params(params, step_result): + """ Evaluates referrences to results from previous steps + """ + def eval_param(param, STEP): + """ Helper function + """ + if isinstance(param, str): + tmp_param = '' + # evaluate every #STEP reference inside parameter itself + for chunk in param.split('#'): + if chunk.startswith('STEP['): + tmp_param = tmp_param + str(eval(chunk)) + else: + tmp_param = tmp_param + chunk + return tmp_param + elif isinstance(param, list) or isinstance(param, tuple): + tmp_list = [] + for item in param: + tmp_list.append(eval_param(item, STEP)) + return tmp_list + elif isinstance(param, dict): + tmp_dict = {} + for (key, value) in param.items(): + tmp_dict[key] = eval_param(value, STEP) + return tmp_dict + else: + return param + + eval_params = [] + # evaluate all parameters if needed + for param in params: + eval_params.append(eval_param(param, step_result)) + return eval_params + + # prepare test execution environment + self.run_initialize() + + with self._vswitch_ctl, self._loadgen: + with self._vnf_ctl, self._collector: + if not self._vswitch_none: + self._add_flows() + + # run traffic generator if requested, otherwise wait for manual termination + if S.getValue('mode') == 'trafficgen-off': + time.sleep(2) + self._logger.debug("All is set. Please run traffic generator manually.") + input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep) + else: + with self._traffic_ctl: + if not self.test: + self._traffic_ctl.send_traffic(self._traffic) + else: + # execute test based on TestSteps definition + if self.test: + step_result = [None] * len(self.test) + for i, step in enumerate(self.test): + step_ok = False + if step[0] == 'vswitch': + test_object = self._vswitch_ctl.get_vswitch() + elif step[0] == 'trafficgen': + test_object = self._traffic_ctl + else: + self._logger.error("Unsupported test object %s", step[0]) + self._inttest = {'status' : False, 'details' : ' '.join(step)} + self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status']) + break + + test_method = getattr(test_object, step[1]) + test_method_check = getattr(test_object, CHECK_PREFIX + step[1]) + + step_params = [] + if test_method and test_method_check and \ + callable(test_method) and callable(test_method_check): + + try: + step_params = eval_step_params(step[2:], step_result) + step_log = '{} {}'.format(' '.join(step[:2]), step_params) + step_result[i] = test_method(*step_params) + self._logger.debug("Step {} '{}' results '{}'".format( + i, step_log, step_result[i])) + time.sleep(2) + step_ok = test_method_check(step_result[i], *step_params) + except AssertionError: + self._inttest = {'status' : False, 'details' : step_log} + self._logger.error("Step {} raised assertion error".format(i)) + break + except IndexError: + self._inttest = {'status' : False, 'details' : step_log} + self._logger.error("Step {} result index error {}".format( + i, ' '.join(step[2:]))) + break + + self.report_status("Step {} - '{}'".format(i, step_log), step_ok) + if not step_ok: + self._inttest = {'status' : False, 'details' : step_log} + break + + # dump vswitch flows before they are affected by VNF termination + if not self._vswitch_none: + self._vswitch_ctl.dump_vswitch_flows() + + # tear down test execution environment and log results + self.run_finalize() + + # report test results + self.run_report() + + def run_report(self): + """ Report test results + """ + if self.test: + results = OrderedDict() + results['status'] = 'OK' if self._inttest['status'] else 'FAILED' + results['details'] = self._inttest['details'] + TestCase._write_result_to_file([results], self._output_file) + self.report_status("Test '{}'".format(self.name), self._inttest['status']) + # inform vsperf about testcase failure + if not self._inttest['status']: + raise Exception diff --git a/testcases/performance.py b/testcases/performance.py new file mode 100644 index 00000000..0ae3ea77 --- /dev/null +++ b/testcases/performance.py @@ -0,0 +1,38 @@ +# Copyright 2015-2016 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PerformanceTestCase class +""" + +import logging + +from testcases import TestCase +from tools.report import report +from conf import settings as S + +class PerformanceTestCase(TestCase): + """PerformanceTestCase class + + In this basic form runs RFC2544 throughput test + """ + def __init__(self, cfg, results_dir): + """ Testcase initialization + """ + self._type = 'performance' + super(PerformanceTestCase, self).__init__(cfg, results_dir) + self._logger = logging.getLogger(__name__) + + def run_report(self): + super(PerformanceTestCase, self).run_report() + if S.getValue('mode') != 'trafficgen-off': + report.generate(self._output_file, self._tc_results, self._collector.get_results(), self._type) diff --git a/testcases/testcase.py b/testcases/testcase.py index dfc766d2..0effce75 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -16,18 +16,17 @@ import csv import os +import time import logging import subprocess import copy -import time from collections import OrderedDict -from core.results.results_constants import ResultsConstants import core.component_factory as component_factory from core.loader import Loader +from core.results.results_constants import ResultsConstants from tools import tasks from tools import hugepages -from tools.report import report from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS from conf import settings as S from conf import get_test_param @@ -37,7 +36,7 @@ class TestCase(object): In this basic form runs RFC2544 throughput test """ - def __init__(self, cfg, results_dir, performance_test=True): + def __init__(self, cfg, results_dir): """Pull out fields from test config :param cfg: A dictionary of string-value pairs describing the test @@ -46,11 +45,19 @@ class TestCase(object): :param results_dir: Where the csv formatted results are written. """ self._hugepages_mounted = False + self._traffic_ctl = None + self._vnf_ctl = None + self._vswitch_ctl = None + self._collector = None + self._loadgen = None + self._output_file = None + self._tc_results = None # set test parameters; CLI options take precedence to testcase settings self._logger = logging.getLogger(__name__) self.name = cfg['Name'] self.desc = cfg.get('Description', 'No description given.') + self.test = cfg.get('TestSteps', None) bidirectional = cfg.get('biDirectional', TRAFFIC_DEFAULTS['bidir']) bidirectional = get_test_param('bidirectional', bidirectional) @@ -63,7 +70,6 @@ class TestCase(object): self.deployment = cfg['Deployment'] self._frame_mod = cfg.get('Frame Modification', None) - self._performance_test = performance_test self._tunnel_type = None self._tunnel_operation = None @@ -131,10 +137,8 @@ class TestCase(object): # Packet Forwarding mode self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower() - def run(self): - """Run the test - - All setup and teardown through controllers is included. + def run_initialize(self): + """ Prepare test execution environment """ self._logger.debug(self.name) @@ -163,35 +167,67 @@ class TestCase(object): self._logger.debug("Controllers:") loader = Loader() - traffic_ctl = component_factory.create_traffic( + self._traffic_ctl = component_factory.create_traffic( self._traffic['traffic_type'], loader.get_trafficgen_class()) - vnf_ctl = component_factory.create_vnf( + + self._vnf_ctl = component_factory.create_vnf( self.deployment, loader.get_vnf_class()) if self._vswitch_none: - vswitch_ctl = component_factory.create_pktfwd( + self._vswitch_ctl = component_factory.create_pktfwd( loader.get_pktfwd_class()) else: - vswitch_ctl = component_factory.create_vswitch( + self._vswitch_ctl = component_factory.create_vswitch( self.deployment, loader.get_vswitch_class(), self._traffic, self._tunnel_operation) - collector = component_factory.create_collector( + self._collector = component_factory.create_collector( loader.get_collector_class(), self._results_dir, self.name) - loadgen = component_factory.create_loadgen( + self._loadgen = component_factory.create_loadgen( self._loadgen, self._load_cfg) + self._output_file = os.path.join(self._results_dir, "result_" + self.name + + "_" + self.deployment + ".csv") + self._logger.debug("Setup:") - with vswitch_ctl, loadgen: - with vnf_ctl, collector: + + def run_finalize(self): + """ Tear down test execution environment and record test results + """ + # umount hugepages if mounted + self._umount_hugepages() + + def run_report(self): + """ Report test results + """ + self._logger.debug("self._collector Results:") + self._collector.print_results() + + if S.getValue('mode') != 'trafficgen-off': + self._logger.debug("Traffic Results:") + self._traffic_ctl.print_results() + + self._tc_results = self._append_results(self._traffic_ctl.get_results()) + TestCase._write_result_to_file(self._tc_results, self._output_file) + + def run(self): + """Run the test + + All setup and teardown through controllers is included. + """ + # prepare test execution environment + self.run_initialize() + + with self._vswitch_ctl, self._loadgen: + with self._vnf_ctl, self._collector: if not self._vswitch_none: - self._add_flows(vswitch_ctl) + self._add_flows() # run traffic generator if requested, otherwise wait for manual termination if S.getValue('mode') == 'trafficgen-off': @@ -209,30 +245,18 @@ class TestCase(object): print('Please respond with \'yes\' or \'y\' ', end='') else: break - with traffic_ctl: - traffic_ctl.send_traffic(self._traffic) + with self._traffic_ctl: + self._traffic_ctl.send_traffic(self._traffic) # dump vswitch flows before they are affected by VNF termination if not self._vswitch_none: - vswitch_ctl.dump_vswitch_flows() + self._vswitch_ctl.dump_vswitch_flows() - # umount hugepages if mounted - self._umount_hugepages() + # tear down test execution environment and log results + self.run_finalize() - self._logger.debug("Collector Results:") - collector.print_results() - - if S.getValue('mode') != 'trafficgen-off': - self._logger.debug("Traffic Results:") - traffic_ctl.print_results() - - output_file = os.path.join(self._results_dir, "result_" + self.name + - "_" + self.deployment + ".csv") - - tc_results = self._append_results(traffic_ctl.get_results()) - TestCase._write_result_to_file(tc_results, output_file) - - report.generate(output_file, tc_results, collector.get_results(), self._performance_test) + # report test results + self.run_report() def _append_results(self, results): """ @@ -335,7 +359,6 @@ class TestCase(object): for result in results: writer.writerow(result) - @staticmethod def _get_unique_keys(list_of_dicts): """Gets unique key values as ordered list of strings in given dicts @@ -351,13 +374,10 @@ class TestCase(object): return list(result.keys()) - - def _add_flows(self, vswitch_ctl): + def _add_flows(self): """Add flows to the vswitch - - :param vswitch_ctl vswitch controller """ - vswitch = vswitch_ctl.get_vswitch() + vswitch = self._vswitch_ctl.get_vswitch() # TODO BOM 15-08-07 the frame mod code assumes that the # physical ports are ports 1 & 2. The actual numbers # need to be retrived from the vSwitch and the metadata value |