From 60cd25286de20e218533c6da6e3dbe29f8644798 Mon Sep 17 00:00:00 2001 From: SerenaFeng Date: Sat, 8 Oct 2016 16:45:49 +0800 Subject: separate visualization building from dashboard building JIRA: RELENG-155 Change-Id: I19355898d5dc5ef73fcfe4ebaf4604c4bf1e8f10 Signed-off-by: SerenaFeng --- dashboard/dashboard/common/elastic_access.py | 6 + dashboard/dashboard/conf/config.py | 12 +- .../elastic2kibana/dashboard_assembler.py | 59 +++++++ dashboard/dashboard/elastic2kibana/main.py | 195 ++++----------------- dashboard/dashboard/elastic2kibana/utility.py | 15 ++ .../elastic2kibana/visualization_assembler.py | 107 +++++++++++ dashboard/dashboard/mongo2elastic/main.py | 4 +- 7 files changed, 225 insertions(+), 173 deletions(-) create mode 100644 dashboard/dashboard/elastic2kibana/dashboard_assembler.py create mode 100644 dashboard/dashboard/elastic2kibana/utility.py create mode 100644 dashboard/dashboard/elastic2kibana/visualization_assembler.py (limited to 'dashboard') diff --git a/dashboard/dashboard/common/elastic_access.py b/dashboard/dashboard/common/elastic_access.py index 8c6494d..aaf776f 100644 --- a/dashboard/dashboard/common/elastic_access.py +++ b/dashboard/dashboard/common/elastic_access.py @@ -1,4 +1,5 @@ import json +import urlparse import urllib3 @@ -43,3 +44,8 @@ def get_docs(url, creds=None, body=None, field='_source'): for hit in res_data['hits']['hits']: docs.append(hit[field]) return docs + + +def publish_kibana(url, creds, type, id, body): + url = urlparse.urljoin(url, '/.kibana/{}/{}'.format(type, id)) + publish_docs(url, creds, body) diff --git a/dashboard/dashboard/conf/config.py b/dashboard/dashboard/conf/config.py index b868999..143b193 100644 --- a/dashboard/dashboard/conf/config.py +++ b/dashboard/dashboard/conf/config.py @@ -23,8 +23,8 @@ class APIConfig: def __init__(self): self._default_config_location = "../etc/config.ini" - self.elastic_url = 'http://localhost:9200' - self.elastic_creds = None + self.es_url = 'http://localhost:9200' + self.es_creds = None self.kibana_url = None self.is_js = True self.js_path = None @@ -64,8 +64,8 @@ class APIConfig: raise ParseError("%s not found" % config_location) # Linking attributes to keys from file with their sections - obj.elastic_url = obj._get_str_parameter("elastic", "url") - obj.elastic_creds = obj._get_str_parameter("elastic", "creds") + obj.es_url = obj._get_str_parameter("elastic", "url") + obj.es_creds = obj._get_str_parameter("elastic", "creds") obj.kibana_url = obj._get_str_parameter("kibana", "url") obj.is_js = obj._get_bool_parameter("kibana", "js") obj.js_path = obj._get_str_parameter("kibana", "js_path") @@ -77,8 +77,8 @@ class APIConfig: "elastic_creds = %s \n" \ "kibana_url = %s \n" \ "is_js = %s \n" \ - "js_path = %s \n" % (self.elastic_url, - self.elastic_creds, + "js_path = %s \n" % (self.es_url, + self.es_creds, self.kibana_url, self.is_js, self.js_path) diff --git a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py new file mode 100644 index 0000000..c1e9dfb --- /dev/null +++ b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py @@ -0,0 +1,59 @@ +import json + +import utility +from common import elastic_access + + +class DashboardAssembler(object): + def __init__(self, + project, + case, + family, + installer, + pod, + visAssemblers, + es_url, + es_creds): + super(DashboardAssembler, self).__init__() + self.project = project + self.case = case + self.test_family = family + self.installer = installer + self.pod = pod + self.visAssemblers = visAssemblers + self.es_url = es_url + self.es_creds = es_creds + self._assemble() + self._publish() + + def _assemble(self): + db = { + "query": { + "project_name": self.project, + "case_name": self.case, + "installer": self.installer, + "metric": self.visAssemblers[0].vis_state_title, + "pod": self.pod + }, + "test_family": self.test_family, + "ids": [visualization.id for visualization in self.visAssemblers] + } + template = utility.env.get_template('dashboard.json') + self.dashboard = json.loads(template.render(db=db)) + utility.dumps(self.dashboard, + ['description', + 'uiStateJSON', + 'panelsJSON', + 'optionsJSON']) + utility.dumps_2depth(self.dashboard, + 'kibanaSavedObjectMeta', + 'searchSourceJSON') + self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-') + return self + + def _publish(self): + elastic_access.publish_kibana(self.es_url, + self.es_creds, + 'dashboard', + self.id, + self.dashboard) diff --git a/dashboard/dashboard/elastic2kibana/main.py b/dashboard/dashboard/elastic2kibana/main.py index ae5cbe8..8be0a01 100644 --- a/dashboard/dashboard/elastic2kibana/main.py +++ b/dashboard/dashboard/elastic2kibana/main.py @@ -3,12 +3,13 @@ import json import urlparse import argparse -from jinja2 import PackageLoader, Environment from common import elastic_access from common import logger_utils +from conf import config from conf import testcases -from conf.config import APIConfig +from dashboard_assembler import DashboardAssembler +from visualization_assembler import VisualizationsAssembler logger = logger_utils.DashboardLogger('elastic2kibana').get @@ -18,154 +19,10 @@ parser.add_argument("-c", "--config-file", help="Config file location") args = parser.parse_args() -CONF = APIConfig().parse(args.config_file) -base_elastic_url = CONF.elastic_url -generate_inputs = CONF.is_js -input_file_path = CONF.js_path -kibana_url = CONF.kibana_url -es_creds = CONF.elastic_creds +CONF = config.APIConfig().parse(args.config_file) _installers = {'fuel', 'apex', 'compass', 'joid'} -env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) -env.filters['jsonify'] = json.dumps - - -def dumps(self, items): - for key in items: - self.visualization[key] = json.dumps(self.visualization[key]) - - -def dumps_2depth(self, key1, key2): - self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2]) - - -class Dashboard(dict): - def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization): - super(Dashboard, self).__init__() - self.project_name = project_name - self.case_name = case_name - self.family = family - self.installer = installer - self.pod = pod - self.scenarios = scenarios - self.visualization = visualization - self._visualization_title = None - self._kibana_visualizations = [] - self._kibana_dashboard = None - self._create_visualizations() - self._create() - - def _create_visualizations(self): - for scenario in self.scenarios: - self._kibana_visualizations.append(Visualization(self.project_name, - self.case_name, - self.installer, - self.pod, - scenario, - self.visualization)) - - self._visualization_title = self._kibana_visualizations[0].vis_state_title - - def _publish_visualizations(self): - for visualization in self._kibana_visualizations: - url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) - logger.debug("publishing visualization '{}'".format(url)) - # logger.error("_publish_visualization: %s" % visualization) - elastic_access.publish_docs(url, es_creds, visualization) - - def _create(self): - db = { - "query": { - "project_name": self.project_name, - "case_name": self.case_name, - "installer": self.installer, - "metric": self._visualization_title, - "pod": self.pod - }, - "test_family": self.family, - "ids": [visualization.id for visualization in self._kibana_visualizations] - } - template = env.get_template('dashboard.json') - self.dashboard = json.loads(template.render(db=db)) - dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON']) - dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-') - - - def _publish(self): - url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) - logger.debug("publishing dashboard '{}'".format(url)) - #logger.error("dashboard: %s" % json.dumps(self.dashboard)) - elastic_access.publish_docs(url, es_creds, self.dashboard) - - def publish(self): - self._publish_visualizations() - self._publish() - - -class VisStateBuilder(object): - def __init__(self, visualization): - super(VisStateBuilder, self).__init__() - self.visualization = visualization - - def build(self): - name = self.visualization.get('name') - fields = self.visualization.get('fields') - - aggs = [] - index = 1 - for field in fields: - aggs.append({ - "id": index, - "field": field.get("field") - }) - index += 1 - - template = env.get_template('{}.json'.format(name)) - vis = template.render(aggs=aggs) - return json.loads(vis) - - -class Visualization(object): - def __init__(self, project_name, case_name, installer, pod, scenario, visualization): - """ - We need two things - 1. filter created from - project_name - case_name - installer - pod - scenario - 2. visualization state - field for y axis (metric) with type (avg, sum, etc.) - field for x axis (segment) with type (date_histogram) - - :return: - """ - super(Visualization, self).__init__() - visState = VisStateBuilder(visualization).build() - self.vis_state_title = visState['title'] - - vis = { - "visState": json.dumps(visState), - "filters": { - "project_name": project_name, - "case_name": case_name, - "installer": installer, - "metric": self.vis_state_title, - "pod_name": pod, - "scenario": scenario - } - } - - template = env.get_template('visualization.json') - - self.visualization = json.loads(template.render(vis=vis)) - dumps(self.visualization, ['visState', 'description', 'uiStateJSON']) - dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.visualization['title'].replace(' ', '-').replace('/', '-') - def _get_pods_and_scenarios(project_name, case_name, installer): query_json = json.JSONEncoder().encode({ @@ -183,8 +40,8 @@ def _get_pods_and_scenarios(project_name, case_name, installer): } }) - elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), - es_creds, + elastic_data = elastic_access.get_docs(urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'), + CONF.es_creds, query_json) pods_and_scenarios = {} @@ -213,24 +70,35 @@ def construct_dashboards(): :return: list of KibanaDashboards """ - kibana_dashboards = [] + dashboards = [] for project, case_dicts in testcases.testcases_yaml.items(): for case in case_dicts: case_name = case.get('name') - visualizations = case.get('visualizations') + vis_ps = case.get('visualizations') family = case.get('test_family') for installer in _installers: pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) - for visualization in visualizations: + for vis_p in vis_ps: for pod, scenarios in pods_and_scenarios.iteritems(): - kibana_dashboards.append(Dashboard(project, - case_name, - family, - installer, - pod, - scenarios, - visualization)) - return kibana_dashboards + vissAssember = VisualizationsAssembler(project, + case_name, + installer, + pod, + scenarios, + vis_p, + CONF.es_url, + CONF.es_creds) + dashboardAssembler = DashboardAssembler(project, + case_name, + family, + installer, + pod, + vissAssember.visAssemblers, + CONF.es_url, + CONF.es_creds) + dashboards.append(dashboardAssembler) + + return dashboards def generate_js_inputs(js_file_path, kibana_url, dashboards): @@ -264,8 +132,5 @@ def generate_js_inputs(js_file_path, kibana_url, dashboards): def main(): dashboards = construct_dashboards() - for kibana_dashboard in dashboards: - kibana_dashboard.publish() - - if generate_inputs: - generate_js_inputs(input_file_path, kibana_url, dashboards) + if CONF.is_js: + generate_js_inputs(CONF.js_path, CONF.kibana_url, dashboards) diff --git a/dashboard/dashboard/elastic2kibana/utility.py b/dashboard/dashboard/elastic2kibana/utility.py new file mode 100644 index 0000000..dccd28a --- /dev/null +++ b/dashboard/dashboard/elastic2kibana/utility.py @@ -0,0 +1,15 @@ +import json + +from jinja2 import Environment, PackageLoader + +env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) +env.filters['jsonify'] = json.dumps + + +def dumps(a_dict, items): + for key in items: + a_dict[key] = json.dumps(a_dict[key]) + + +def dumps_2depth(a_dict, key1, key2): + a_dict[key1][key2] = json.dumps(a_dict[key1][key2]) diff --git a/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/dashboard/dashboard/elastic2kibana/visualization_assembler.py new file mode 100644 index 0000000..e3b6b0d --- /dev/null +++ b/dashboard/dashboard/elastic2kibana/visualization_assembler.py @@ -0,0 +1,107 @@ +import json + +import utility +from common import elastic_access + + +class VisStateBuilder(object): + def __init__(self, vis_p): + super(VisStateBuilder, self).__init__() + self.vis_p = vis_p + + def build(self): + name = self.vis_p.get('name') + fields = self.vis_p.get('fields') + + aggs = [] + index = 1 + for field in fields: + aggs.append({ + "id": index, + "field": field.get("field") + }) + index += 1 + + template = utility.env.get_template('{}.json'.format(name)) + vis = template.render(aggs=aggs) + return json.loads(vis) + + +class VisualizationAssembler(object): + def __init__(self, + project, + case, + installer, + pod, + scenario, + vis_p, + es_url, + es_creds): + super(VisualizationAssembler, self).__init__() + self.project = project + self.case = case + self.installer = installer + self.pod = pod + self.scenario = scenario + self.vis_p = vis_p + self.es_url = es_url + self.es_creds = es_creds + self._assemble() + self._publish() + + def _assemble(self): + visState = VisStateBuilder(self.vis_p).build() + self.vis_state_title = visState['title'] + + vis = { + "visState": json.dumps(visState), + "filters": { + "project_name": self.project, + "case_name": self.case, + "installer": self.installer, + "metric": self.vis_state_title, + "pod_name": self.pod, + "scenario": self.scenario + } + } + + template = utility.env.get_template('visualization.json') + + self.visualization = json.loads(template.render(vis=vis)) + utility.dumps(self.visualization, + ['visState', 'description', 'uiStateJSON']) + utility.dumps_2depth(self.visualization, + 'kibanaSavedObjectMeta', + 'searchSourceJSON') + title = self.visualization['title'] + self.id = title.replace(' ', '-').replace('/', '-') + + def _publish(self): + elastic_access.publish_kibana(self.es_url, + self.es_creds, + 'visualization', + self.id, + self.visualization) + + +class VisualizationsAssembler(object): + def __init__(self, + project, + case, + installer, + pod, + scenarios, + vis_p, + es_url, + es_creds): + super(VisualizationsAssembler, self).__init__() + self.visAssemblers = [] + for scenario in scenarios: + self.visAssemblers.append(VisualizationAssembler(project, + case, + installer, + pod, + scenario, + vis_p, + es_url, + es_creds)) diff --git a/dashboard/dashboard/mongo2elastic/main.py b/dashboard/dashboard/mongo2elastic/main.py index 76efb14..b13f8a7 100644 --- a/dashboard/dashboard/mongo2elastic/main.py +++ b/dashboard/dashboard/mongo2elastic/main.py @@ -250,9 +250,9 @@ class DocumentsPublisher: def main(): - base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic') + base_elastic_url = urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic') days = args.latest_days - es_creds = CONF.elastic_creds + es_creds = CONF.es_creds for project, case_dicts in testcases.testcases_yaml.items(): for case_dict in case_dicts: -- cgit 1.2.3-korg