From f2e764a50b598eced8806fc0da1b3a21b7d91ec3 Mon Sep 17 00:00:00 2001 From: SerenaFeng Date: Mon, 17 Oct 2016 09:03:38 +0800 Subject: project-ize kibana_dashboard JIRA: FUNCTEST-513 Change-Id: Id25dbea72b5945bc40f30c0a4b3ffe3898c3ab0b Signed-off-by: SerenaFeng --- dashboard/dashboard/conf/config.py | 2 +- dashboard/dashboard/conf/testcases.py | 2 +- .../elastic2kibana/dashboard_assembler.py | 2 +- dashboard/dashboard/elastic2kibana/main.py | 8 +- dashboard/dashboard/elastic2kibana/utility.py | 2 +- .../elastic2kibana/visualization_assembler.py | 2 +- dashboard/dashboard/elastic2kibana_main.py | 4 - dashboard/dashboard/functest/format.py | 186 +++++++++++++++++++++ dashboard/dashboard/mongo2elastic/main.py | 9 +- dashboard/dashboard/mongo2elastic_main.py | 4 - dashboard/dashboard/qtip/format.py | 19 +++ 11 files changed, 219 insertions(+), 21 deletions(-) delete mode 100644 dashboard/dashboard/elastic2kibana_main.py create mode 100644 dashboard/dashboard/functest/format.py delete mode 100644 dashboard/dashboard/mongo2elastic_main.py create mode 100644 dashboard/dashboard/qtip/format.py (limited to 'dashboard/dashboard') diff --git a/dashboard/dashboard/conf/config.py b/dashboard/dashboard/conf/config.py index 48fed88..44cf797 100644 --- a/dashboard/dashboard/conf/config.py +++ b/dashboard/dashboard/conf/config.py @@ -22,7 +22,7 @@ class APIConfig: """ def __init__(self): - self._default_config_location = "../etc/config.ini" + self._default_config_location = "/etc/dashboard/config.ini" self.es_url = 'http://localhost:9200' self.es_creds = None self.kibana_url = None diff --git a/dashboard/dashboard/conf/testcases.py b/dashboard/dashboard/conf/testcases.py index e120987..ff801b4 100644 --- a/dashboard/dashboard/conf/testcases.py +++ b/dashboard/dashboard/conf/testcases.py @@ -1,7 +1,7 @@ import yaml -with open('./functest/testcases.yaml') as f: +with open('/etc/dashboard/testcases.yaml') as f: testcases_yaml = yaml.safe_load(f) f.close() diff --git a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py index da7ccfc..651168b 100644 --- a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py +++ b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py @@ -1,7 +1,7 @@ import json import utility -from common import elastic_access +from dashboard.common import elastic_access class DashboardAssembler(object): diff --git a/dashboard/dashboard/elastic2kibana/main.py b/dashboard/dashboard/elastic2kibana/main.py index 9ee8942..e4c17d7 100644 --- a/dashboard/dashboard/elastic2kibana/main.py +++ b/dashboard/dashboard/elastic2kibana/main.py @@ -4,10 +4,10 @@ import urlparse import argparse -from common import elastic_access -from common import logger_utils -from conf import config -from conf import testcases +from dashboard.common import elastic_access +from dashboard.common import logger_utils +from dashboard.conf import config +from dashboard.conf import testcases from dashboard_assembler import DashboardAssembler from visualization_assembler import VisualizationAssembler diff --git a/dashboard/dashboard/elastic2kibana/utility.py b/dashboard/dashboard/elastic2kibana/utility.py index dccd28a..55578bd 100644 --- a/dashboard/dashboard/elastic2kibana/utility.py +++ b/dashboard/dashboard/elastic2kibana/utility.py @@ -2,7 +2,7 @@ import json from jinja2 import Environment, PackageLoader -env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) +env = Environment(loader=PackageLoader('dashboard', 'elastic2kibana/templates')) env.filters['jsonify'] = json.dumps diff --git a/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/dashboard/dashboard/elastic2kibana/visualization_assembler.py index 1cb0ba8..d7e6e54 100644 --- a/dashboard/dashboard/elastic2kibana/visualization_assembler.py +++ b/dashboard/dashboard/elastic2kibana/visualization_assembler.py @@ -1,7 +1,7 @@ import json import utility -from common import elastic_access +from dashboard.common import elastic_access class VisStateBuilder(object): diff --git a/dashboard/dashboard/elastic2kibana_main.py b/dashboard/dashboard/elastic2kibana_main.py deleted file mode 100644 index 3ec27cb..0000000 --- a/dashboard/dashboard/elastic2kibana_main.py +++ /dev/null @@ -1,4 +0,0 @@ -from elastic2kibana.main import main - -if __name__ == '__main__': - main() diff --git a/dashboard/dashboard/functest/format.py b/dashboard/dashboard/functest/format.py new file mode 100644 index 0000000..ef485ba --- /dev/null +++ b/dashboard/dashboard/functest/format.py @@ -0,0 +1,186 @@ +#! /usr/bin/env python + + +def _convert_value(value): + return value if value != '' else 0 + + +def _convert_duration(duration): + if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration: + hours, minutes, seconds = duration.split(":") + hours = _convert_value(hours) + minutes = _convert_value(minutes) + seconds = _convert_value(seconds) + int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds) + else: + int_duration = duration + return int_duration + + +def format_normal(testcase): + """ + Look for these and leave any of those: + details.duration + details.tests + details.failures + + If none are present, then return False + """ + found = False + testcase_details = testcase['details'] + fields = ['duration', 'tests', 'failures'] + if isinstance(testcase_details, dict): + for key, value in testcase_details.items(): + if key in fields: + found = True + if key == 'duration': + testcase_details[key] = _convert_duration(value) + else: + del testcase_details[key] + + if 'tests' in testcase_details and 'failures' in testcase_details: + testcase_tests = float(testcase_details['tests']) + testcase_failures = float(testcase_details['failures']) + if testcase_tests != 0: + testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests + else: + testcase_details['success_percentage'] = 0 + + + return found + + +def format_rally(testcase): + """ + Structure: + details.[{summary.duration}] + details.[{summary.nb success}] + details.[{summary.nb tests}] + + Find data for these fields + -> details.duration + -> details.tests + -> details.success_percentage + """ + details = testcase['details'] + summary = None + for item in details: + if 'summary' in item: + summary = item['summary'] + + if not summary: + return False + + testcase['details'] = { + 'duration': summary['duration'], + 'tests': summary['nb tests'], + 'success_percentage': summary['nb success'] + } + return True + + +def _get_statistics(orig_data, stat_fields, stat_values=None): + test_results = {} + for stat_data in orig_data: + for field in stat_fields: + stat_value = stat_data[field] + if stat_value in test_results: + test_results[stat_value] += 1 + else: + test_results[stat_value] = 1 + + if stat_values is not None: + for stat_value in stat_values: + if stat_value not in test_results: + test_results[stat_value] = 0 + + return test_results + + +def format_onos(testcase): + """ + Structure: + details.FUNCvirNet.duration + details.FUNCvirNet.status.[{Case result}] + details.FUNCvirNetL3.duration + details.FUNCvirNetL3.status.[{Case result}] + + Find data for these fields + -> details.FUNCvirNet.duration + -> details.FUNCvirNet.tests + -> details.FUNCvirNet.failures + -> details.FUNCvirNetL3.duration + -> details.FUNCvirNetL3.tests + -> details.FUNCvirNetL3.failures + """ + testcase_details = testcase['details'] + + if 'FUNCvirNet' not in testcase_details or 'FUNCvirNetL3' not in testcase_details: + return False + + funcvirnet_details = testcase_details['FUNCvirNet']['status'] + funcvirnet_stats = _get_statistics(funcvirnet_details, ('Case result',), ('PASS', 'FAIL')) + funcvirnet_passed = funcvirnet_stats['PASS'] + funcvirnet_failed = funcvirnet_stats['FAIL'] + funcvirnet_all = funcvirnet_passed + funcvirnet_failed + + funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status'] + funcvirnetl3_stats = _get_statistics(funcvirnetl3_details, ('Case result',), ('PASS', 'FAIL')) + funcvirnetl3_passed = funcvirnetl3_stats['PASS'] + funcvirnetl3_failed = funcvirnetl3_stats['FAIL'] + funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed + + testcase_details['FUNCvirNet'] = { + 'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']), + 'tests': funcvirnet_all, + 'failures': funcvirnet_failed + } + testcase_details['FUNCvirNetL3'] = { + 'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']), + 'tests': funcvirnetl3_all, + 'failures': funcvirnetl3_failed + } + return True + + +def format_vims(testcase): + """ + Structure: + details.sig_test.result.[{result}] + details.sig_test.duration + details.vIMS.duration + details.orchestrator.duration + + Find data for these fields + -> details.sig_test.duration + -> details.sig_test.tests + -> details.sig_test.failures + -> details.sig_test.passed + -> details.sig_test.skipped + -> details.vIMS.duration + -> details.orchestrator.duration + """ + testcase_details = testcase['details'] + test_results = _get_statistics(testcase_details['sig_test']['result'], + ('result',), + ('Passed', 'Skipped', 'Failed')) + passed = test_results['Passed'] + skipped = test_results['Skipped'] + failures = test_results['Failed'] + all_tests = passed + skipped + failures + testcase['details'] = { + 'sig_test': { + 'duration': testcase_details['sig_test']['duration'], + 'tests': all_tests, + 'failures': failures, + 'passed': passed, + 'skipped': skipped + }, + 'vIMS': { + 'duration': testcase_details['vIMS']['duration'] + }, + 'orchestrator': { + 'duration': testcase_details['orchestrator']['duration'] + } + } + return True diff --git a/dashboard/dashboard/mongo2elastic/main.py b/dashboard/dashboard/mongo2elastic/main.py index 303d82c..3f92c56 100644 --- a/dashboard/dashboard/mongo2elastic/main.py +++ b/dashboard/dashboard/mongo2elastic/main.py @@ -10,10 +10,11 @@ import uuid import argparse -from common import logger_utils, elastic_access -from conf import testcases -from conf.config import APIConfig -from mongo2elastic import format +from dashboard.common import elastic_access +from dashboard.common import logger_utils +from dashboard.conf import testcases +from dashboard.conf.config import APIConfig +from dashboard.mongo2elastic import format logger = logger_utils.DashboardLogger('mongo2elastic').get diff --git a/dashboard/dashboard/mongo2elastic_main.py b/dashboard/dashboard/mongo2elastic_main.py deleted file mode 100644 index 141d8f3..0000000 --- a/dashboard/dashboard/mongo2elastic_main.py +++ /dev/null @@ -1,4 +0,0 @@ -from mongo2elastic.main import main - -if __name__ == '__main__': - main() diff --git a/dashboard/dashboard/qtip/format.py b/dashboard/dashboard/qtip/format.py new file mode 100644 index 0000000..b78fa5b --- /dev/null +++ b/dashboard/dashboard/qtip/format.py @@ -0,0 +1,19 @@ +#! /usr/bin/env python + + +def format_qpi(testcase): + """ + Look for these and leave any of those: + details.index + + If none are present, then return False + """ + details = testcase['details'] + if 'index' not in details: + return False + + for key, value in details.items(): + if key != 'index': + del details[key] + + return True -- cgit 1.2.3-korg