diff options
Diffstat (limited to 'functest/opnfv_tests/openstack/rally/rally.py')
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 140 |
1 files changed, 82 insertions, 58 deletions
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 8d1bfabb5..16699a019 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -181,14 +181,14 @@ class RallyBase(testcase.TestCase): :return: Bool """ rally_report = json.loads(json_raw) - for report in rally_report: - if report is None or report.get('result') is None: - return False - - for result in report.get('result'): - if result is None or result.get('error'): + tasks = rally_report.get('tasks') + if tasks: + for task in tasks: + if task.get('status') != 'finished' or \ + task.get('pass_sla') is not True: return False - + else: + return False return True def _migration_supported(self): @@ -316,6 +316,52 @@ class RallyBase(testcase.TestCase): return True + def _save_results(self, test_name, task_id): + """ Generate and save task execution results""" + # check for result directory and create it otherwise + if not os.path.exists(self.RESULTS_DIR): + LOGGER.debug('%s does not exist, we create it.', + self.RESULTS_DIR) + os.makedirs(self.RESULTS_DIR) + + # put detailed result to log + cmd = (["rally", "task", "detailed", "--uuid", task_id]) + LOGGER.debug('running command: %s', cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + json_detailed = self.get_cmd_output(proc) + LOGGER.info('%s', json_detailed) + + # save report as JSON + cmd = (["rally", "task", "report", "--json", "--uuid", task_id]) + LOGGER.debug('running command: %s', cmd) + with open(os.devnull, 'w') as devnull: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=devnull) + json_results = self.get_cmd_output(proc) + report_json_name = 'opnfv-{}.json'.format(test_name) + report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name) + with open(report_json_dir, 'w') as r_file: + LOGGER.debug('saving json file') + r_file.write(json_results) + + # save report as HTML + report_html_name = 'opnfv-{}.html'.format(test_name) + report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name) + cmd = (["rally", "task", "report", "--html", "--uuid", task_id, + "--out", report_html_dir]) + LOGGER.debug('running command: %s', cmd) + with open(os.devnull, 'w') as devnull: + subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull) + + self._append_summary(json_results, test_name) + + # parse JSON operation result + if self.task_succeed(json_results): + LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n") + else: + LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n") + def _run_task(self, test_name): """Run a task.""" LOGGER.info('Starting test scenario "%s" ...', test_name) @@ -351,47 +397,9 @@ class RallyBase(testcase.TestCase): stderr=subprocess.STDOUT) output = self.get_cmd_output(proc) LOGGER.error("Task validation result:" + "\n" + output) - return + raise Exception("Failed to retrieve task id") - # check for result directory and create it otherwise - if not os.path.exists(self.RESULTS_DIR): - LOGGER.debug('%s does not exist, we create it.', - self.RESULTS_DIR) - os.makedirs(self.RESULTS_DIR) - - # get and save rally operation JSON result - cmd = (["rally", "task", "detailed", task_id]) - LOGGER.debug('running command: %s', cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - json_detailed = self.get_cmd_output(proc) - LOGGER.info('%s', json_detailed) - - cmd = (["rally", "task", "results", task_id]) - LOGGER.debug('running command: %s', cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - json_results = self.get_cmd_output(proc) - self._append_summary(json_results, test_name) - report_json_name = 'opnfv-{}.json'.format(test_name) - report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name) - with open(report_json_dir, 'w') as r_file: - LOGGER.debug('saving json file') - r_file.write(json_results) - - # write html report file - report_html_name = 'opnfv-{}.html'.format(test_name) - report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name) - cmd = (["rally", "task", "report", task_id, "--out", report_html_dir]) - LOGGER.debug('running command: %s', cmd) - subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # parse JSON operation result - if self.task_succeed(json_results): - LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n") - else: - LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n") + self._save_results(test_name, task_id) def _append_summary(self, json_raw, test_name): """Update statistics summary info.""" @@ -400,20 +408,24 @@ class RallyBase(testcase.TestCase): overall_duration = 0.0 rally_report = json.loads(json_raw) - for report in rally_report: - if report.get('full_duration'): - overall_duration += report.get('full_duration') + for task in rally_report.get('tasks'): + for subtask in task.get('subtasks'): + for workload in subtask.get('workloads'): + if workload.get('full_duration'): + overall_duration += workload.get('full_duration') - if report.get('result'): - for result in report.get('result'): - nb_tests += 1 - if not result.get('error'): - nb_success += 1 + if workload.get('data'): + nb_tests += len(workload.get('data')) + + for result in workload.get('data'): + if not result.get('error'): + nb_success += 1 scenario_summary = {'test_name': test_name, 'overall_duration': overall_duration, 'nb_tests': nb_tests, - 'nb_success': nb_success} + 'nb_success': nb_success, + 'task_status': self.task_succeed(json_raw)} self.summary.append(scenario_summary) def _prepare_env(self): @@ -508,6 +520,7 @@ class RallyBase(testcase.TestCase): total_duration = 0.0 total_nb_tests = 0 total_nb_success = 0 + nb_modules = 0 payload = [] res_table = prettytable.PrettyTable( @@ -519,6 +532,8 @@ class RallyBase(testcase.TestCase): # for each scenario we draw a row for the table for item in self.summary: + if item['task_status'] is True: + nb_modules += 1 total_duration += item['overall_duration'] total_nb_tests += item['nb_tests'] total_nb_success += item['nb_success'] @@ -549,8 +564,9 @@ class RallyBase(testcase.TestCase): success_rate_str]) LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string()) - LOGGER.info("Rally '%s' success_rate is %s%%", - self.case_name, success_rate) + LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules", + self.case_name, success_rate, nb_modules, + len(self.summary)) payload.append({'summary': {'duration': total_duration, 'nb tests': total_nb_tests, 'nb success': success_rate}}) @@ -573,6 +589,14 @@ class RallyBase(testcase.TestCase): if self.image: self.cloud.delete_image(self.image.id) + def is_successful(self): + """The overall result of the test.""" + for item in self.summary: + if item['task_status'] is False: + return testcase.TestCase.EX_TESTCASE_FAILED + + return super(RallyBase, self).is_successful() + @energy.enable_recording def run(self, **kwargs): """Run testcase.""" |