diff options
Diffstat (limited to 'dashboard/dashboard/elastic2kibana/main.py')
| -rw-r--r-- | dashboard/dashboard/elastic2kibana/main.py | 195 |
1 files changed, 30 insertions, 165 deletions
diff --git a/dashboard/dashboard/elastic2kibana/main.py b/dashboard/dashboard/elastic2kibana/main.py index ae5cbe8..8be0a01 100644 --- a/dashboard/dashboard/elastic2kibana/main.py +++ b/dashboard/dashboard/elastic2kibana/main.py @@ -3,12 +3,13 @@ import json import urlparse import argparse -from jinja2 import PackageLoader, Environment from common import elastic_access from common import logger_utils +from conf import config from conf import testcases -from conf.config import APIConfig +from dashboard_assembler import DashboardAssembler +from visualization_assembler import VisualizationsAssembler logger = logger_utils.DashboardLogger('elastic2kibana').get @@ -18,154 +19,10 @@ parser.add_argument("-c", "--config-file", help="Config file location") args = parser.parse_args() -CONF = APIConfig().parse(args.config_file) -base_elastic_url = CONF.elastic_url -generate_inputs = CONF.is_js -input_file_path = CONF.js_path -kibana_url = CONF.kibana_url -es_creds = CONF.elastic_creds +CONF = config.APIConfig().parse(args.config_file) _installers = {'fuel', 'apex', 'compass', 'joid'} -env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) -env.filters['jsonify'] = json.dumps - - -def dumps(self, items): - for key in items: - self.visualization[key] = json.dumps(self.visualization[key]) - - -def dumps_2depth(self, key1, key2): - self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2]) - - -class Dashboard(dict): - def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization): - super(Dashboard, self).__init__() - self.project_name = project_name - self.case_name = case_name - self.family = family - self.installer = installer - self.pod = pod - self.scenarios = scenarios - self.visualization = visualization - self._visualization_title = None - self._kibana_visualizations = [] - self._kibana_dashboard = None - self._create_visualizations() - self._create() - - def _create_visualizations(self): - for scenario in self.scenarios: - self._kibana_visualizations.append(Visualization(self.project_name, - self.case_name, - self.installer, - self.pod, - scenario, - self.visualization)) - - self._visualization_title = self._kibana_visualizations[0].vis_state_title - - def _publish_visualizations(self): - for visualization in self._kibana_visualizations: - url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) - logger.debug("publishing visualization '{}'".format(url)) - # logger.error("_publish_visualization: %s" % visualization) - elastic_access.publish_docs(url, es_creds, visualization) - - def _create(self): - db = { - "query": { - "project_name": self.project_name, - "case_name": self.case_name, - "installer": self.installer, - "metric": self._visualization_title, - "pod": self.pod - }, - "test_family": self.family, - "ids": [visualization.id for visualization in self._kibana_visualizations] - } - template = env.get_template('dashboard.json') - self.dashboard = json.loads(template.render(db=db)) - dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON']) - dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-') - - - def _publish(self): - url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) - logger.debug("publishing dashboard '{}'".format(url)) - #logger.error("dashboard: %s" % json.dumps(self.dashboard)) - elastic_access.publish_docs(url, es_creds, self.dashboard) - - def publish(self): - self._publish_visualizations() - self._publish() - - -class VisStateBuilder(object): - def __init__(self, visualization): - super(VisStateBuilder, self).__init__() - self.visualization = visualization - - def build(self): - name = self.visualization.get('name') - fields = self.visualization.get('fields') - - aggs = [] - index = 1 - for field in fields: - aggs.append({ - "id": index, - "field": field.get("field") - }) - index += 1 - - template = env.get_template('{}.json'.format(name)) - vis = template.render(aggs=aggs) - return json.loads(vis) - - -class Visualization(object): - def __init__(self, project_name, case_name, installer, pod, scenario, visualization): - """ - We need two things - 1. filter created from - project_name - case_name - installer - pod - scenario - 2. visualization state - field for y axis (metric) with type (avg, sum, etc.) - field for x axis (segment) with type (date_histogram) - - :return: - """ - super(Visualization, self).__init__() - visState = VisStateBuilder(visualization).build() - self.vis_state_title = visState['title'] - - vis = { - "visState": json.dumps(visState), - "filters": { - "project_name": project_name, - "case_name": case_name, - "installer": installer, - "metric": self.vis_state_title, - "pod_name": pod, - "scenario": scenario - } - } - - template = env.get_template('visualization.json') - - self.visualization = json.loads(template.render(vis=vis)) - dumps(self.visualization, ['visState', 'description', 'uiStateJSON']) - dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.visualization['title'].replace(' ', '-').replace('/', '-') - def _get_pods_and_scenarios(project_name, case_name, installer): query_json = json.JSONEncoder().encode({ @@ -183,8 +40,8 @@ def _get_pods_and_scenarios(project_name, case_name, installer): } }) - elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), - es_creds, + elastic_data = elastic_access.get_docs(urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'), + CONF.es_creds, query_json) pods_and_scenarios = {} @@ -213,24 +70,35 @@ def construct_dashboards(): :return: list of KibanaDashboards """ - kibana_dashboards = [] + dashboards = [] for project, case_dicts in testcases.testcases_yaml.items(): for case in case_dicts: case_name = case.get('name') - visualizations = case.get('visualizations') + vis_ps = case.get('visualizations') family = case.get('test_family') for installer in _installers: pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) - for visualization in visualizations: + for vis_p in vis_ps: for pod, scenarios in pods_and_scenarios.iteritems(): - kibana_dashboards.append(Dashboard(project, - case_name, - family, - installer, - pod, - scenarios, - visualization)) - return kibana_dashboards + vissAssember = VisualizationsAssembler(project, + case_name, + installer, + pod, + scenarios, + vis_p, + CONF.es_url, + CONF.es_creds) + dashboardAssembler = DashboardAssembler(project, + case_name, + family, + installer, + pod, + vissAssember.visAssemblers, + CONF.es_url, + CONF.es_creds) + dashboards.append(dashboardAssembler) + + return dashboards def generate_js_inputs(js_file_path, kibana_url, dashboards): @@ -264,8 +132,5 @@ def generate_js_inputs(js_file_path, kibana_url, dashboards): def main(): dashboards = construct_dashboards() - for kibana_dashboard in dashboards: - kibana_dashboard.publish() - - if generate_inputs: - generate_js_inputs(input_file_path, kibana_url, dashboards) + if CONF.is_js: + generate_js_inputs(CONF.js_path, CONF.kibana_url, dashboards) |