diff options
4 files changed, 123 insertions, 124 deletions
diff --git a/utils/test/dashboard/dashboard/common/logger_utils.py b/utils/test/dashboard/dashboard/common/logger_utils.py index 183080810..58e343d79 100644 --- a/utils/test/dashboard/dashboard/common/logger_utils.py +++ b/utils/test/dashboard/dashboard/common/logger_utils.py @@ -62,4 +62,3 @@ class DashboardLogger(Logger): def __init__(self, logger_name): super(DashboardLogger, self).__init__(logger_name) - diff --git a/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py index c1e9dfb22..da7ccfc24 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py @@ -11,7 +11,7 @@ class DashboardAssembler(object): family, installer, pod, - visAssemblers, + visualizations, es_url, es_creds): super(DashboardAssembler, self).__init__() @@ -20,7 +20,7 @@ class DashboardAssembler(object): self.test_family = family self.installer = installer self.pod = pod - self.visAssemblers = visAssemblers + self.visualizations = visualizations self.es_url = es_url self.es_creds = es_creds self._assemble() @@ -32,11 +32,11 @@ class DashboardAssembler(object): "project_name": self.project, "case_name": self.case, "installer": self.installer, - "metric": self.visAssemblers[0].vis_state_title, + "metric": self.visualizations[0].vis_state_title, "pod": self.pod }, "test_family": self.test_family, - "ids": [visualization.id for visualization in self.visAssemblers] + "ids": [visualization.id for visualization in self.visualizations] } template = utility.env.get_template('dashboard.json') self.dashboard = json.loads(template.render(db=db)) diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py index 8be0a01dd..35a154307 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/main.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py @@ -9,7 +9,7 @@ from common import logger_utils from conf import config from conf import testcases from dashboard_assembler import DashboardAssembler -from visualization_assembler import VisualizationsAssembler +from visualization_assembler import VisualizationAssembler logger = logger_utils.DashboardLogger('elastic2kibana').get @@ -24,113 +24,136 @@ CONF = config.APIConfig().parse(args.config_file) _installers = {'fuel', 'apex', 'compass', 'joid'} -def _get_pods_and_scenarios(project_name, case_name, installer): - query_json = json.JSONEncoder().encode({ - "query": { - "bool": { - "must": [ - {"match_all": {}} - ], - "filter": [ - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}} - ] - } - } - }) - - elastic_data = elastic_access.get_docs(urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'), - CONF.es_creds, - query_json) - - pods_and_scenarios = {} - - for data in elastic_data: - pod = data['pod_name'] - if pod in pods_and_scenarios: - pods_and_scenarios[pod].add(data['scenario']) - else: - pods_and_scenarios[pod] = {data['scenario']} - - if 'all' in pods_and_scenarios: - pods_and_scenarios['all'].add(data['scenario']) - else: - pods_and_scenarios['all'] = {data['scenario']} - - return pods_and_scenarios - - -def construct_dashboards(): - """ - iterate over testcase and installer - 1. get available pods for each testcase/installer pair - 2. get available scenario for each testcase/installer/pod tuple - 3. construct KibanaInput and append - - :return: list of KibanaDashboards - """ - dashboards = [] - for project, case_dicts in testcases.testcases_yaml.items(): - for case in case_dicts: - case_name = case.get('name') - vis_ps = case.get('visualizations') - family = case.get('test_family') - for installer in _installers: - pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) - for vis_p in vis_ps: - for pod, scenarios in pods_and_scenarios.iteritems(): - vissAssember = VisualizationsAssembler(project, - case_name, - installer, - pod, - scenarios, - vis_p, - CONF.es_url, - CONF.es_creds) - dashboardAssembler = DashboardAssembler(project, - case_name, - family, +class KibanaConstructor(object): + def __init__(self): + super(KibanaConstructor, self).__init__() + self.js_dict = {} + + def construct(self): + for project, case_dicts in testcases.testcases_yaml.items(): + for case in case_dicts: + self._construct_by_case(project, case) + return self + + def _construct_by_case(self, project, case): + case_name = case.get('name') + vis_ps = case.get('visualizations') + family = case.get('test_family') + for vis_p in vis_ps: + self._construct_by_vis(project, case_name, family, vis_p) + + def _construct_by_vis(self, project, case, family, vis_p): + for installer in _installers: + pods_and_scenarios = self._get_pods_and_scenarios(project, + case, + installer) + for pod, scenarios in pods_and_scenarios.iteritems(): + visualizations = self._construct_visualizations(project, + case, installer, pod, - vissAssember.visAssemblers, + scenarios, + vis_p, CONF.es_url, CONF.es_creds) - dashboards.append(dashboardAssembler) - - return dashboards - - -def generate_js_inputs(js_file_path, kibana_url, dashboards): - js_dict = {} - for dashboard in dashboards: - dashboard_meta = dashboard.dashboard['metadata'] - test_family = dashboard_meta['test_family'] - test_label = dashboard_meta['label'] - - if test_family not in js_dict: - js_dict[test_family] = {} - - js_test_family = js_dict[test_family] + dashboard = DashboardAssembler(project, + case, + family, + installer, + pod, + visualizations, + CONF.es_url, + CONF.es_creds) + if CONF.is_js: + self._set_js_dict(case, + pod, + installer, + family, + vis_p.get('name'), + dashboard.id) + + @staticmethod + def _construct_visualizations(project, + case, + installer, + pod, + scenarios, + vis_p, + es_url, + es_creds): + visualizations = [] + for scenario in scenarios: + visualizations.append(VisualizationAssembler(project, + case, + installer, + pod, + scenario, + vis_p, + es_url, + es_creds)) + return visualizations + + def _set_js_dict(self, case, pod, installer, family, metric, id): + test_label = '{} {}'.format(case, metric) + if family not in self.js_dict: + self.js_dict[family] = {} + + js_test_family = self.js_dict[family] if test_label not in js_test_family: js_test_family[test_label] = {} js_test_label = js_test_family[test_label] - if dashboard.installer not in js_test_label: - js_test_label[dashboard.installer] = {} + if installer not in js_test_label: + js_test_label[installer] = {} + + js_installer = js_test_label[installer] + js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id + + def config_js(self): + if CONF.is_js: + with open(CONF.js_path, 'w+') as conf_js_fdesc: + conf_js_fdesc.write('var kibana_dashboard_links = ') + conf_js_fdesc.write(str(self.js_dict).replace("u'", "'")) + + def _get_pods_and_scenarios(self, project, case, installer): + query = json.JSONEncoder().encode({ + "query": { + "bool": { + "must": [ + {"match_all": {}} + ], + "filter": [ + {"match": {"installer": installer}}, + {"match": {"project_name": project}}, + {"match": {"case_name": case}} + ] + } + } + }) - js_installer = js_test_label[dashboard.installer] - js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id + elastic_data = elastic_access.get_docs( + urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'), + CONF.es_creds, + query) - with open(js_file_path, 'w+') as js_file_fdesc: - js_file_fdesc.write('var kibana_dashboard_links = ') - js_file_fdesc.write(str(js_dict).replace("u'", "'")) + pods_and_scenarios = {} + for data in elastic_data: + pod = data['pod_name'] + if pod in pods_and_scenarios: + pods_and_scenarios[pod].add(data['scenario']) + else: + pods_and_scenarios[pod] = {data['scenario']} -def main(): - dashboards = construct_dashboards() + if 'all' in pods_and_scenarios: + pods_and_scenarios['all'].add(data['scenario']) + else: + pods_and_scenarios['all'] = {data['scenario']} + + return pods_and_scenarios - if CONF.is_js: - generate_js_inputs(CONF.js_path, CONF.kibana_url, dashboards) + +def main(): + KibanaConstructor().construct().config_js() diff --git a/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py index e3b6b0d38..1cb0ba8d1 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py @@ -82,26 +82,3 @@ class VisualizationAssembler(object): 'visualization', self.id, self.visualization) - - -class VisualizationsAssembler(object): - def __init__(self, - project, - case, - installer, - pod, - scenarios, - vis_p, - es_url, - es_creds): - super(VisualizationsAssembler, self).__init__() - self.visAssemblers = [] - for scenario in scenarios: - self.visAssemblers.append(VisualizationAssembler(project, - case, - installer, - pod, - scenario, - vis_p, - es_url, - es_creds)) |