diff options
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 140 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/rally/test_rally.py | 65 |
2 files changed, 130 insertions, 75 deletions
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 4d12a617a..7d91720a3 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -151,14 +151,14 @@ class RallyBase(singlevm.VmReady1): :return: Bool """ rally_report = json.loads(json_raw) - for report in rally_report: - if report is None or report.get('result') is None: - return False - - for result in report.get('result'): - if result is None or result.get('error'): + tasks = rally_report.get('tasks') + if tasks: + for task in tasks: + if task.get('status') != 'finished' or \ + task.get('pass_sla') is not True: return False - + else: + return False return True def _migration_supported(self): @@ -286,6 +286,52 @@ class RallyBase(singlevm.VmReady1): return True + def _save_results(self, test_name, task_id): + """ Generate and save task execution results""" + # check for result directory and create it otherwise + if not os.path.exists(self.RESULTS_DIR): + LOGGER.debug('%s does not exist, we create it.', + self.RESULTS_DIR) + os.makedirs(self.RESULTS_DIR) + + # put detailed result to log + cmd = (["rally", "task", "detailed", "--uuid", task_id]) + LOGGER.debug('running command: %s', cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + json_detailed = self.get_cmd_output(proc) + LOGGER.info('%s', json_detailed) + + # save report as JSON + cmd = (["rally", "task", "report", "--json", "--uuid", task_id]) + LOGGER.debug('running command: %s', cmd) + with open(os.devnull, 'w') as devnull: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=devnull) + json_results = self.get_cmd_output(proc) + report_json_name = 'opnfv-{}.json'.format(test_name) + report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name) + with open(report_json_dir, 'w') as r_file: + LOGGER.debug('saving json file') + r_file.write(json_results) + + # save report as HTML + report_html_name = 'opnfv-{}.html'.format(test_name) + report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name) + cmd = (["rally", "task", "report", "--html", "--uuid", task_id, + "--out", report_html_dir]) + LOGGER.debug('running command: %s', cmd) + with open(os.devnull, 'w') as devnull: + subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull) + + self._append_summary(json_results, test_name) + + # parse JSON operation result + if self.task_succeed(json_results): + LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n") + else: + LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n") + def _run_task(self, test_name): """Run a task.""" LOGGER.info('Starting test scenario "%s" ...', test_name) @@ -321,47 +367,9 @@ class RallyBase(singlevm.VmReady1): stderr=subprocess.STDOUT) output = self.get_cmd_output(proc) LOGGER.error("Task validation result:" + "\n" + output) - return + raise Exception("Failed to retrieve task id") - # check for result directory and create it otherwise - if not os.path.exists(self.RESULTS_DIR): - LOGGER.debug('%s does not exist, we create it.', - self.RESULTS_DIR) - os.makedirs(self.RESULTS_DIR) - - # get and save rally operation JSON result - cmd = (["rally", "task", "detailed", task_id]) - LOGGER.debug('running command: %s', cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - json_detailed = self.get_cmd_output(proc) - LOGGER.info('%s', json_detailed) - - cmd = (["rally", "task", "results", task_id]) - LOGGER.debug('running command: %s', cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - json_results = self.get_cmd_output(proc) - self._append_summary(json_results, test_name) - report_json_name = 'opnfv-{}.json'.format(test_name) - report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name) - with open(report_json_dir, 'w') as r_file: - LOGGER.debug('saving json file') - r_file.write(json_results) - - # write html report file - report_html_name = 'opnfv-{}.html'.format(test_name) - report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name) - cmd = (["rally", "task", "report", task_id, "--out", report_html_dir]) - LOGGER.debug('running command: %s', cmd) - subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # parse JSON operation result - if self.task_succeed(json_results): - LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n") - else: - LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n") + self._save_results(test_name, task_id) def _append_summary(self, json_raw, test_name): """Update statistics summary info.""" @@ -370,20 +378,24 @@ class RallyBase(singlevm.VmReady1): overall_duration = 0.0 rally_report = json.loads(json_raw) - for report in rally_report: - if report.get('full_duration'): - overall_duration += report.get('full_duration') + for task in rally_report.get('tasks'): + for subtask in task.get('subtasks'): + for workload in subtask.get('workloads'): + if workload.get('full_duration'): + overall_duration += workload.get('full_duration') - if report.get('result'): - for result in report.get('result'): - nb_tests += 1 - if not result.get('error'): - nb_success += 1 + if workload.get('data'): + nb_tests += len(workload.get('data')) + + for result in workload.get('data'): + if not result.get('error'): + nb_success += 1 scenario_summary = {'test_name': test_name, 'overall_duration': overall_duration, 'nb_tests': nb_tests, - 'nb_success': nb_success} + 'nb_success': nb_success, + 'task_status': self.task_succeed(json_raw)} self.summary.append(scenario_summary) def _prepare_env(self): @@ -412,6 +424,7 @@ class RallyBase(singlevm.VmReady1): total_duration = 0.0 total_nb_tests = 0 total_nb_success = 0 + nb_modules = 0 payload = [] res_table = prettytable.PrettyTable( @@ -423,6 +436,8 @@ class RallyBase(singlevm.VmReady1): # for each scenario we draw a row for the table for item in self.summary: + if item['task_status'] is True: + nb_modules += 1 total_duration += item['overall_duration'] total_nb_tests += item['nb_tests'] total_nb_success += item['nb_success'] @@ -453,8 +468,9 @@ class RallyBase(singlevm.VmReady1): success_rate_str]) LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string()) - LOGGER.info("Rally '%s' success_rate is %s%%", - self.case_name, success_rate) + LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules", + self.case_name, success_rate, nb_modules, + len(self.summary)) payload.append({'summary': {'duration': total_duration, 'nb tests': total_nb_tests, 'nb success': success_rate}}) @@ -466,6 +482,14 @@ class RallyBase(singlevm.VmReady1): if self.flavor_alt: self.orig_cloud.delete_flavor(self.flavor_alt.id) + def is_successful(self): + """The overall result of the test.""" + for item in self.summary: + if item['task_status'] is False: + return testcase.TestCase.EX_TESTCASE_FAILED + + return super(RallyBase, self).is_successful() + @energy.enable_recording def run(self, **kwargs): """Run testcase.""" diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index cbcf46fce..e7805c72f 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -90,15 +90,16 @@ class OSRallyTesting(unittest.TestCase): None) def test_task_succeed_fail(self): - json_raw = json.dumps([None]) + json_raw = json.dumps({}) self.assertEqual(self.rally_base.task_succeed(json_raw), False) - json_raw = json.dumps([{'result': [{'error': ['test_error']}]}]) + json_raw = json.dumps({'tasks': [{'status': 'crashed'}]}) self.assertEqual(self.rally_base.task_succeed(json_raw), False) def test_task_succeed_success(self): - json_raw = json.dumps('') + json_raw = json.dumps({'tasks': [{'status': 'finished', + 'pass_sla': True}]}) self.assertEqual(self.rally_base.task_succeed(json_raw), True) @@ -219,8 +220,6 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' '_build_task_args', return_value={}) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - '_append_summary') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'get_task_id', return_value=None) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'get_cmd_output', return_value='') @@ -230,7 +229,8 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error') def test_run_task_taskid_missing(self, mock_logger_error, *args): # pylint: disable=unused-argument - self.rally_base._run_task('test_name') + with self.assertRaises(Exception): + self.rally_base._run_task('test_name') text = 'Failed to retrieve task_id, validating task...' mock_logger_error.assert_any_call(text) @@ -242,8 +242,6 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' '_build_task_args', return_value={}) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - '_append_summary') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'get_task_id', return_value='1') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'get_cmd_output', return_value='') @@ -255,13 +253,30 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info') @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error') - def test_run_task_default(self, mock_logger_error, mock_logger_info, - *args): + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' + '_save_results') + def test_run_task_default(self, mock_save_res, *args): # pylint: disable=unused-argument self.rally_base._run_task('test_name') - text = 'Test scenario: "test_name" OK.\n' - mock_logger_info.assert_any_call(text) - mock_logger_error.assert_not_called() + mock_save_res.assert_called() + + @mock.patch('six.moves.builtins.open', mock.mock_open()) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' + 'task_succeed', return_value=True) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' + 'get_cmd_output', return_value='') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists', + return_value=True) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' + '_append_summary') + def test_save_results(self, mock_summary, *args): + # pylint: disable=unused-argument + self.rally_base._save_results('test_name', '1234') + mock_summary.assert_called() def test_prepare_env_testname_invalid(self): self.rally_base.TESTS = ['test1', 'test2'] @@ -341,15 +356,31 @@ class OSRallyTesting(unittest.TestCase): mock_prep_env.assert_called() def test_append_summary(self): - text = '[{"result":[{"error":[]},{"error":["err"]}],' \ - '"full_duration": 17.312026}]' + text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \ + '1.23,"data": [{"error": []}]}]},{"workloads": ' \ + '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}' self.rally_base._append_summary(text, "foo_test") self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test") - self.assertEqual(self.rally_base.summary[0]['overall_duration'], - 17.312026) + self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01) self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2) self.assertEqual(self.rally_base.summary[0]['nb_success'], 1) + def test_is_successful_false(self): + with mock.patch('six.moves.builtins.super') as mock_super: + self.rally_base.summary = [{"task_status": True}, + {"task_status": False}] + self.assertEqual(self.rally_base.is_successful(), + testcase.TestCase.EX_TESTCASE_FAILED) + mock_super(rally.RallyBase, self).is_successful.assert_not_called() + + def test_is_successful_true(self): + with mock.patch('six.moves.builtins.super') as mock_super: + mock_super(rally.RallyBase, self).is_successful.return_value = 424 + self.rally_base.summary = [{"task_status": True}, + {"task_status": True}] + self.assertEqual(self.rally_base.is_successful(), 424) + mock_super(rally.RallyBase, self).is_successful.assert_called() + if __name__ == "__main__": logging.disable(logging.CRITICAL) |