diff options
Diffstat (limited to 'utils/test/dashboard')
3 files changed, 165 insertions, 139 deletions
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py index 95f758ea8..ae5cbe8fa 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/main.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py @@ -27,10 +27,22 @@ es_creds = CONF.elastic_creds _installers = {'fuel', 'apex', 'compass', 'joid'} +env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) +env.filters['jsonify'] = json.dumps -class KibanaDashboard(dict): + +def dumps(self, items): + for key in items: + self.visualization[key] = json.dumps(self.visualization[key]) + + +def dumps_2depth(self, key1, key2): + self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2]) + + +class Dashboard(dict): def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization): - super(KibanaDashboard, self).__init__() + super(Dashboard, self).__init__() self.project_name = project_name self.case_name = case_name self.family = family @@ -46,14 +58,14 @@ class KibanaDashboard(dict): def _create_visualizations(self): for scenario in self.scenarios: - self._kibana_visualizations.append(KibanaVisualization(self.project_name, - self.case_name, - self.installer, - self.pod, - scenario, - self.visualization)) + self._kibana_visualizations.append(Visualization(self.project_name, + self.case_name, + self.installer, + self.pod, + scenario, + self.visualization)) - self._visualization_title = self._kibana_visualizations[0].vis_title + self._visualization_title = self._kibana_visualizations[0].vis_state_title def _publish_visualizations(self): for visualization in self._kibana_visualizations: @@ -62,113 +74,39 @@ class KibanaDashboard(dict): # logger.error("_publish_visualization: %s" % visualization) elastic_access.publish_docs(url, es_creds, visualization) - def _construct_panels(self): - size_x = 6 - size_y = 3 - max_columns = 7 - column = 1 - row = 1 - panel_index = 1 - panels_json = [] - for visualization in self._kibana_visualizations: - panels_json.append({ - "id": visualization.id, - "type": 'visualization', - "panelIndex": panel_index, - "size_x": size_x, - "size_y": size_y, - "col": column, - "row": row - }) - panel_index += 1 - column += size_x - if column > max_columns: - column = 1 - row += size_y - return json.dumps(panels_json, separators=(',', ':')) - def _create(self): - self['title'] = '{} {} {} {} {}'.format(self.project_name, - self.case_name, - self.installer, - self._visualization_title, - self.pod) - self.id = self['title'].replace(' ', '-').replace('/', '-') - - self['hits'] = 0 - self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \ - " pod '{}'".format(self.project_name, - self.case_name, - self.installer, - self._visualization_title, - self.pod) - self['panelsJSON'] = self._construct_panels() - self['optionsJSON'] = json.dumps({ - "darkTheme": False - }, - separators=(',', ':')) - self['uiStateJSON'] = "{}" - self['scenario'] = 1 - self['timeRestore'] = False - self['kibanaSavedObjectMeta'] = { - 'searchSourceJSON': json.dumps({ - "filter": [ - { - "query": { - "query_string": { - "query": "*", - "analyze_wildcard": True - } - } - } - ] + db = { + "query": { + "project_name": self.project_name, + "case_name": self.case_name, + "installer": self.installer, + "metric": self._visualization_title, + "pod": self.pod }, - separators=(',', ':')) + "test_family": self.family, + "ids": [visualization.id for visualization in self._kibana_visualizations] } + template = env.get_template('dashboard.json') + self.dashboard = json.loads(template.render(db=db)) + dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON']) + dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON') + self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-') - label = self.case_name - if 'label' in self.visualization: - label += " %s" % self.visualization.get('label') - label += " %s" % self.visualization.get('name') - self['metadata'] = { - "label": label, - "test_family": self.family - } def _publish(self): url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) logger.debug("publishing dashboard '{}'".format(url)) - elastic_access.publish_docs(url, es_creds, self) + #logger.error("dashboard: %s" % json.dumps(self.dashboard)) + elastic_access.publish_docs(url, es_creds, self.dashboard) def publish(self): self._publish_visualizations() self._publish() -class KibanaSearchSourceJSON(dict): - """ - "filter": [ - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}} - ] - """ - - def __init__(self, project_name, case_name, installer, pod, scenario): - super(KibanaSearchSourceJSON, self).__init__() - self["filter"] = [ - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}}, - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"scenario": {"query": scenario, "type": "phrase"}}} - ] - if pod != 'all': - self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}}) - - -class VisualizationBuilder(object): +class VisStateBuilder(object): def __init__(self, visualization): - super(VisualizationBuilder, self).__init__() + super(VisStateBuilder, self).__init__() self.visualization = visualization def build(self): @@ -184,14 +122,12 @@ class VisualizationBuilder(object): }) index += 1 - env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) - env.filters['jsonify'] = json.dumps template = env.get_template('{}.json'.format(name)) vis = template.render(aggs=aggs) return json.loads(vis) -class KibanaVisualization(dict): +class Visualization(object): def __init__(self, project_name, case_name, installer, pod, scenario, visualization): """ We need two things @@ -207,32 +143,28 @@ class KibanaVisualization(dict): :return: """ - super(KibanaVisualization, self).__init__() - vis = VisualizationBuilder(visualization).build() - self.vis_title = vis['title'] - self['title'] = '{} {} {} {} {} {}'.format(project_name, - case_name, - self.vis_title, - installer, - pod, - scenario) - self.id = self['title'].replace(' ', '-').replace('/', '-') - self['visState'] = json.dumps(vis, separators=(',', ':')) - self['uiStateJSON'] = "{}" - self['description'] = "Kibana visualization for project_name '{}', case_name '{}', metric '{}', installer '{}'," \ - " pod '{}' and scenario '{}'".format(project_name, - case_name, - self.vis_title, - installer, - pod, - scenario) - self['scenario'] = 1 - self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name, - case_name, - installer, - pod, - scenario), - separators=(',', ':'))} + super(Visualization, self).__init__() + visState = VisStateBuilder(visualization).build() + self.vis_state_title = visState['title'] + + vis = { + "visState": json.dumps(visState), + "filters": { + "project_name": project_name, + "case_name": case_name, + "installer": installer, + "metric": self.vis_state_title, + "pod_name": pod, + "scenario": scenario + } + } + + template = env.get_template('visualization.json') + + self.visualization = json.loads(template.render(vis=vis)) + dumps(self.visualization, ['visState', 'description', 'uiStateJSON']) + dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON') + self.id = self.visualization['title'].replace(' ', '-').replace('/', '-') def _get_pods_and_scenarios(project_name, case_name, installer): @@ -252,7 +184,8 @@ def _get_pods_and_scenarios(project_name, case_name, installer): }) elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), - es_creds, query_json) + es_creds, + query_json) pods_and_scenarios = {} @@ -290,20 +223,20 @@ def construct_dashboards(): pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) for visualization in visualizations: for pod, scenarios in pods_and_scenarios.iteritems(): - kibana_dashboards.append(KibanaDashboard(project, - case_name, - family, - installer, - pod, - scenarios, - visualization)) + kibana_dashboards.append(Dashboard(project, + case_name, + family, + installer, + pod, + scenarios, + visualization)) return kibana_dashboards def generate_js_inputs(js_file_path, kibana_url, dashboards): js_dict = {} for dashboard in dashboards: - dashboard_meta = dashboard['metadata'] + dashboard_meta = dashboard.dashboard['metadata'] test_family = dashboard_meta['test_family'] test_label = dashboard_meta['label'] diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json new file mode 100644 index 000000000..cc8099593 --- /dev/null +++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json @@ -0,0 +1,61 @@ +{% set db = db|default({}) -%} + + +{% macro calc_col(index) -%} + {% if index is divisibleby 2 %} + 7 + {% else %} + 1 + {% endif %} +{%- endmacro %} + +{% macro calc_row(index) -%} +{% set num = (index - 1)//2 %} + {{1 + num * 3}} +{%- endmacro %} + +{ + "description": "Kibana dashboard for {{db.query}}", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + } + ] + } + }, + "metadata": { + "label": "{{db.query.case_name}} {{db.query.metric}}", + "test_family": "{{db.test_family}}" + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + {% for id in db.ids %} + { + "col": {{calc_col(loop.index)}}, + "id": "{{id}}", + "panelIndex": {{loop.index}}, + "row": {{calc_row(loop.index)}}, + "size_x": 6, + "size_y": 3, + "type": "visualization" + } + {% if not loop.last %} + , + {% endif %} + {% endfor %} + ], + "scenario": 1, + "timeRestore": false, + "title": "{{db.query.project_name}} {{db.query.case_name}} {{db.query.installer}} {{db.query.metric}} {{db.query.pod}}", + "uiStateJSON": {} +} diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json new file mode 100644 index 000000000..d51d4174e --- /dev/null +++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json @@ -0,0 +1,32 @@ +{% set vis = vis|default({}) -%} + + +{ + "description": "Kibana visualization for {{ vis.filters }}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + {% for key, value in vis.filters.iteritems() if key != "metric" %} + {% if not (key == "pod_name" and value == "all") %} + { + "match": { + "{{ key }}": { + "query": "{{ value }}", + "type": "phrase" + } + } + } + {% if not loop.last %} + , + {% endif %} + {% endif %} + {% endfor %} + ] + } + }, + "scenario": 1, + "title": "{{vis.filters.project_name}} {{vis.filters.case_name}} {{vis.filters.installer}} {{vis.filters.metric}} {{vis.filters.pod_name}} {{vis.filters.scenario}}", + "uiStateJSON": {}, + "visState": {{ vis.visState }} +} + |