summaryrefslogtreecommitdiffstats
path: root/utils/test/dashboard
diff options
context:
space:
mode:
Diffstat (limited to 'utils/test/dashboard')
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/main.py211
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json61
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json32
3 files changed, 165 insertions, 139 deletions
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py
index 95f758ea8..ae5cbe8fa 100644
--- a/utils/test/dashboard/dashboard/elastic2kibana/main.py
+++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py
@@ -27,10 +27,22 @@ es_creds = CONF.elastic_creds
_installers = {'fuel', 'apex', 'compass', 'joid'}
+env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
+env.filters['jsonify'] = json.dumps
-class KibanaDashboard(dict):
+
+def dumps(self, items):
+ for key in items:
+ self.visualization[key] = json.dumps(self.visualization[key])
+
+
+def dumps_2depth(self, key1, key2):
+ self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
+
+
+class Dashboard(dict):
def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
- super(KibanaDashboard, self).__init__()
+ super(Dashboard, self).__init__()
self.project_name = project_name
self.case_name = case_name
self.family = family
@@ -46,14 +58,14 @@ class KibanaDashboard(dict):
def _create_visualizations(self):
for scenario in self.scenarios:
- self._kibana_visualizations.append(KibanaVisualization(self.project_name,
- self.case_name,
- self.installer,
- self.pod,
- scenario,
- self.visualization))
+ self._kibana_visualizations.append(Visualization(self.project_name,
+ self.case_name,
+ self.installer,
+ self.pod,
+ scenario,
+ self.visualization))
- self._visualization_title = self._kibana_visualizations[0].vis_title
+ self._visualization_title = self._kibana_visualizations[0].vis_state_title
def _publish_visualizations(self):
for visualization in self._kibana_visualizations:
@@ -62,113 +74,39 @@ class KibanaDashboard(dict):
# logger.error("_publish_visualization: %s" % visualization)
elastic_access.publish_docs(url, es_creds, visualization)
- def _construct_panels(self):
- size_x = 6
- size_y = 3
- max_columns = 7
- column = 1
- row = 1
- panel_index = 1
- panels_json = []
- for visualization in self._kibana_visualizations:
- panels_json.append({
- "id": visualization.id,
- "type": 'visualization',
- "panelIndex": panel_index,
- "size_x": size_x,
- "size_y": size_y,
- "col": column,
- "row": row
- })
- panel_index += 1
- column += size_x
- if column > max_columns:
- column = 1
- row += size_y
- return json.dumps(panels_json, separators=(',', ':'))
-
def _create(self):
- self['title'] = '{} {} {} {} {}'.format(self.project_name,
- self.case_name,
- self.installer,
- self._visualization_title,
- self.pod)
- self.id = self['title'].replace(' ', '-').replace('/', '-')
-
- self['hits'] = 0
- self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
- " pod '{}'".format(self.project_name,
- self.case_name,
- self.installer,
- self._visualization_title,
- self.pod)
- self['panelsJSON'] = self._construct_panels()
- self['optionsJSON'] = json.dumps({
- "darkTheme": False
- },
- separators=(',', ':'))
- self['uiStateJSON'] = "{}"
- self['scenario'] = 1
- self['timeRestore'] = False
- self['kibanaSavedObjectMeta'] = {
- 'searchSourceJSON': json.dumps({
- "filter": [
- {
- "query": {
- "query_string": {
- "query": "*",
- "analyze_wildcard": True
- }
- }
- }
- ]
+ db = {
+ "query": {
+ "project_name": self.project_name,
+ "case_name": self.case_name,
+ "installer": self.installer,
+ "metric": self._visualization_title,
+ "pod": self.pod
},
- separators=(',', ':'))
+ "test_family": self.family,
+ "ids": [visualization.id for visualization in self._kibana_visualizations]
}
+ template = env.get_template('dashboard.json')
+ self.dashboard = json.loads(template.render(db=db))
+ dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON'])
+ dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON')
+ self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
- label = self.case_name
- if 'label' in self.visualization:
- label += " %s" % self.visualization.get('label')
- label += " %s" % self.visualization.get('name')
- self['metadata'] = {
- "label": label,
- "test_family": self.family
- }
def _publish(self):
url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
logger.debug("publishing dashboard '{}'".format(url))
- elastic_access.publish_docs(url, es_creds, self)
+ #logger.error("dashboard: %s" % json.dumps(self.dashboard))
+ elastic_access.publish_docs(url, es_creds, self.dashboard)
def publish(self):
self._publish_visualizations()
self._publish()
-class KibanaSearchSourceJSON(dict):
- """
- "filter": [
- {"match": {"installer": {"query": installer, "type": "phrase"}}},
- {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
- {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
- ]
- """
-
- def __init__(self, project_name, case_name, installer, pod, scenario):
- super(KibanaSearchSourceJSON, self).__init__()
- self["filter"] = [
- {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
- {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
- {"match": {"installer": {"query": installer, "type": "phrase"}}},
- {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
- ]
- if pod != 'all':
- self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
-
-
-class VisualizationBuilder(object):
+class VisStateBuilder(object):
def __init__(self, visualization):
- super(VisualizationBuilder, self).__init__()
+ super(VisStateBuilder, self).__init__()
self.visualization = visualization
def build(self):
@@ -184,14 +122,12 @@ class VisualizationBuilder(object):
})
index += 1
- env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
- env.filters['jsonify'] = json.dumps
template = env.get_template('{}.json'.format(name))
vis = template.render(aggs=aggs)
return json.loads(vis)
-class KibanaVisualization(dict):
+class Visualization(object):
def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
"""
We need two things
@@ -207,32 +143,28 @@ class KibanaVisualization(dict):
:return:
"""
- super(KibanaVisualization, self).__init__()
- vis = VisualizationBuilder(visualization).build()
- self.vis_title = vis['title']
- self['title'] = '{} {} {} {} {} {}'.format(project_name,
- case_name,
- self.vis_title,
- installer,
- pod,
- scenario)
- self.id = self['title'].replace(' ', '-').replace('/', '-')
- self['visState'] = json.dumps(vis, separators=(',', ':'))
- self['uiStateJSON'] = "{}"
- self['description'] = "Kibana visualization for project_name '{}', case_name '{}', metric '{}', installer '{}'," \
- " pod '{}' and scenario '{}'".format(project_name,
- case_name,
- self.vis_title,
- installer,
- pod,
- scenario)
- self['scenario'] = 1
- self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
- case_name,
- installer,
- pod,
- scenario),
- separators=(',', ':'))}
+ super(Visualization, self).__init__()
+ visState = VisStateBuilder(visualization).build()
+ self.vis_state_title = visState['title']
+
+ vis = {
+ "visState": json.dumps(visState),
+ "filters": {
+ "project_name": project_name,
+ "case_name": case_name,
+ "installer": installer,
+ "metric": self.vis_state_title,
+ "pod_name": pod,
+ "scenario": scenario
+ }
+ }
+
+ template = env.get_template('visualization.json')
+
+ self.visualization = json.loads(template.render(vis=vis))
+ dumps(self.visualization, ['visState', 'description', 'uiStateJSON'])
+ dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON')
+ self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
def _get_pods_and_scenarios(project_name, case_name, installer):
@@ -252,7 +184,8 @@ def _get_pods_and_scenarios(project_name, case_name, installer):
})
elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
- es_creds, query_json)
+ es_creds,
+ query_json)
pods_and_scenarios = {}
@@ -290,20 +223,20 @@ def construct_dashboards():
pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
for visualization in visualizations:
for pod, scenarios in pods_and_scenarios.iteritems():
- kibana_dashboards.append(KibanaDashboard(project,
- case_name,
- family,
- installer,
- pod,
- scenarios,
- visualization))
+ kibana_dashboards.append(Dashboard(project,
+ case_name,
+ family,
+ installer,
+ pod,
+ scenarios,
+ visualization))
return kibana_dashboards
def generate_js_inputs(js_file_path, kibana_url, dashboards):
js_dict = {}
for dashboard in dashboards:
- dashboard_meta = dashboard['metadata']
+ dashboard_meta = dashboard.dashboard['metadata']
test_family = dashboard_meta['test_family']
test_label = dashboard_meta['label']
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json
new file mode 100644
index 000000000..cc8099593
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/dashboard.json
@@ -0,0 +1,61 @@
+{% set db = db|default({}) -%}
+
+
+{% macro calc_col(index) -%}
+ {% if index is divisibleby 2 %}
+ 7
+ {% else %}
+ 1
+ {% endif %}
+{%- endmacro %}
+
+{% macro calc_row(index) -%}
+{% set num = (index - 1)//2 %}
+ {{1 + num * 3}}
+{%- endmacro %}
+
+{
+ "description": "Kibana dashboard for {{db.query}}",
+ "hits": 0,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": {
+ "filter": [
+ {
+ "query": {
+ "query_string": {
+ "analyze_wildcard": true,
+ "query": "*"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "metadata": {
+ "label": "{{db.query.case_name}} {{db.query.metric}}",
+ "test_family": "{{db.test_family}}"
+ },
+ "optionsJSON": {
+ "darkTheme": false
+ },
+ "panelsJSON": [
+ {% for id in db.ids %}
+ {
+ "col": {{calc_col(loop.index)}},
+ "id": "{{id}}",
+ "panelIndex": {{loop.index}},
+ "row": {{calc_row(loop.index)}},
+ "size_x": 6,
+ "size_y": 3,
+ "type": "visualization"
+ }
+ {% if not loop.last %}
+ ,
+ {% endif %}
+ {% endfor %}
+ ],
+ "scenario": 1,
+ "timeRestore": false,
+ "title": "{{db.query.project_name}} {{db.query.case_name}} {{db.query.installer}} {{db.query.metric}} {{db.query.pod}}",
+ "uiStateJSON": {}
+}
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json
new file mode 100644
index 000000000..d51d4174e
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/visualization.json
@@ -0,0 +1,32 @@
+{% set vis = vis|default({}) -%}
+
+
+{
+ "description": "Kibana visualization for {{ vis.filters }}",
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": {
+ "filter": [
+ {% for key, value in vis.filters.iteritems() if key != "metric" %}
+ {% if not (key == "pod_name" and value == "all") %}
+ {
+ "match": {
+ "{{ key }}": {
+ "query": "{{ value }}",
+ "type": "phrase"
+ }
+ }
+ }
+ {% if not loop.last %}
+ ,
+ {% endif %}
+ {% endif %}
+ {% endfor %}
+ ]
+ }
+ },
+ "scenario": 1,
+ "title": "{{vis.filters.project_name}} {{vis.filters.case_name}} {{vis.filters.installer}} {{vis.filters.metric}} {{vis.filters.pod_name}} {{vis.filters.scenario}}",
+ "uiStateJSON": {},
+ "visState": {{ vis.visState }}
+}
+