diff options
Diffstat (limited to 'dashboard/dashboard/elastic2kibana')
11 files changed, 0 insertions, 587 deletions
diff --git a/dashboard/dashboard/elastic2kibana/__init__.py b/dashboard/dashboard/elastic2kibana/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/dashboard/dashboard/elastic2kibana/__init__.py +++ /dev/null diff --git a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py deleted file mode 100644 index 651168b..0000000 --- a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py +++ /dev/null @@ -1,59 +0,0 @@ -import json - -import utility -from dashboard.common import elastic_access - - -class DashboardAssembler(object): - def __init__(self, - project, - case, - family, - installer, - pod, - visualizations, - es_url, - es_creds): - super(DashboardAssembler, self).__init__() - self.project = project - self.case = case - self.test_family = family - self.installer = installer - self.pod = pod - self.visualizations = visualizations - self.es_url = es_url - self.es_creds = es_creds - self._assemble() - self._publish() - - def _assemble(self): - db = { - "query": { - "project_name": self.project, - "case_name": self.case, - "installer": self.installer, - "metric": self.visualizations[0].vis_state_title, - "pod": self.pod - }, - "test_family": self.test_family, - "ids": [visualization.id for visualization in self.visualizations] - } - template = utility.env.get_template('dashboard.json') - self.dashboard = json.loads(template.render(db=db)) - utility.dumps(self.dashboard, - ['description', - 'uiStateJSON', - 'panelsJSON', - 'optionsJSON']) - utility.dumps_2depth(self.dashboard, - 'kibanaSavedObjectMeta', - 'searchSourceJSON') - self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-') - return self - - def _publish(self): - elastic_access.publish_kibana(self.es_url, - self.es_creds, - 'dashboard', - self.id, - self.dashboard) diff --git a/dashboard/dashboard/elastic2kibana/main.py b/dashboard/dashboard/elastic2kibana/main.py deleted file mode 100644 index 112d222..0000000 --- a/dashboard/dashboard/elastic2kibana/main.py +++ /dev/null @@ -1,155 +0,0 @@ -#! /usr/bin/env python -import json - -import argparse - -from dashboard.common import elastic_access -from dashboard.common import logger_utils -from dashboard.conf import config -from dashboard.conf import testcases -from dashboard_assembler import DashboardAssembler -from visualization_assembler import VisualizationAssembler - -logger = logger_utils.DashboardLogger('elastic2kibana').get - -parser = argparse.ArgumentParser() -parser.add_argument("-c", "--config-file", - dest='config_file', - help="Config file location") - -args = parser.parse_args() -CONF = config.APIConfig().parse(args.config_file) - -_installers = {'fuel', 'apex', 'compass', 'joid'} - - -class KibanaConstructor(object): - def __init__(self): - super(KibanaConstructor, self).__init__() - self.js_dict = {} - - def construct(self): - for project, case_dicts in testcases.testcases_yaml.items(): - for case in case_dicts: - self._construct_by_case(project, case) - return self - - def _construct_by_case(self, project, case): - case_name = case.get('name') - vis_ps = case.get('visualizations') - family = case.get('test_family') - for vis_p in vis_ps: - self._construct_by_vis(project, case_name, family, vis_p) - - def _construct_by_vis(self, project, case, family, vis_p): - for installer in _installers: - pods_and_scenarios = self._get_pods_and_scenarios(project, - case, - installer) - for pod, scenarios in pods_and_scenarios.iteritems(): - visualizations = self._construct_visualizations(project, - case, - installer, - pod, - scenarios, - vis_p, - CONF.es_url, - CONF.es_creds) - dashboard = DashboardAssembler(project, - case, - family, - installer, - pod, - visualizations, - CONF.es_url, - CONF.es_creds) - self._set_js_dict(case, - pod, - installer, - family, - vis_p.get('name'), - dashboard.id) - - @staticmethod - def _construct_visualizations(project, - case, - installer, - pod, - scenarios, - vis_p, - es_url, - es_creds): - visualizations = [] - for scenario in scenarios: - visualizations.append(VisualizationAssembler(project, - case, - installer, - pod, - scenario, - vis_p, - es_url, - es_creds)) - return visualizations - - def _set_js_dict(self, case, pod, installer, family, metric, id): - test_label = '{} {}'.format(case, metric) - if family not in self.js_dict: - self.js_dict[family] = {} - - js_test_family = self.js_dict[family] - - if test_label not in js_test_family: - js_test_family[test_label] = {} - - js_test_label = js_test_family[test_label] - - if installer not in js_test_label: - js_test_label[installer] = {} - - js_installer = js_test_label[installer] - js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id - - def config_js(self): - with open(CONF.js_path, 'w+') as conf_js_fdesc: - conf_js_fdesc.write('var kibana_dashboard_links = ') - conf_js_fdesc.write(str(self.js_dict).replace("u'", "'")) - - def _get_pods_and_scenarios(self, project, case, installer): - query = json.JSONEncoder().encode({ - "query": { - "bool": { - "must": [ - {"match_all": {}} - ], - "filter": [ - {"match": {"installer": installer}}, - {"match": {"project_name": project}}, - {"match": {"case_name": case}} - ] - } - } - }) - - elastic_data = elastic_access.get_docs(CONF.index_url, - CONF.es_creds, - query) - - pods_and_scenarios = {} - - for data in elastic_data: - pod = data['pod_name'] - if pod in pods_and_scenarios: - pods_and_scenarios[pod].add(data['scenario']) - else: - pods_and_scenarios[pod] = {data['scenario']} - - if 'all' in pods_and_scenarios: - pods_and_scenarios['all'].add(data['scenario']) - else: - pods_and_scenarios['all'] = {data['scenario']} - - return pods_and_scenarios - - -def main(): - KibanaConstructor().construct().config_js() diff --git a/dashboard/dashboard/elastic2kibana/templates/dashboard.json b/dashboard/dashboard/elastic2kibana/templates/dashboard.json deleted file mode 100644 index cc80995..0000000 --- a/dashboard/dashboard/elastic2kibana/templates/dashboard.json +++ /dev/null @@ -1,61 +0,0 @@ -{% set db = db|default({}) -%} - - -{% macro calc_col(index) -%} - {% if index is divisibleby 2 %} - 7 - {% else %} - 1 - {% endif %} -{%- endmacro %} - -{% macro calc_row(index) -%} -{% set num = (index - 1)//2 %} - {{1 + num * 3}} -{%- endmacro %} - -{ - "description": "Kibana dashboard for {{db.query}}", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - } - ] - } - }, - "metadata": { - "label": "{{db.query.case_name}} {{db.query.metric}}", - "test_family": "{{db.test_family}}" - }, - "optionsJSON": { - "darkTheme": false - }, - "panelsJSON": [ - {% for id in db.ids %} - { - "col": {{calc_col(loop.index)}}, - "id": "{{id}}", - "panelIndex": {{loop.index}}, - "row": {{calc_row(loop.index)}}, - "size_x": 6, - "size_y": 3, - "type": "visualization" - } - {% if not loop.last %} - , - {% endif %} - {% endfor %} - ], - "scenario": 1, - "timeRestore": false, - "title": "{{db.query.project_name}} {{db.query.case_name}} {{db.query.installer}} {{db.query.metric}} {{db.query.pod}}", - "uiStateJSON": {} -} diff --git a/dashboard/dashboard/elastic2kibana/templates/duration.json b/dashboard/dashboard/elastic2kibana/templates/duration.json deleted file mode 100644 index f50a668..0000000 --- a/dashboard/dashboard/elastic2kibana/templates/duration.json +++ /dev/null @@ -1,45 +0,0 @@ -{% set aggs = aggs|default([]) -%} - -{ - "title": "duration", - "type": "line", - "listeners": {}, - "params": { - "addLegend": true, - "shareYAxis": true, - "addTooltip": true, - "smoothLines": false, - "scale": "linear", - "interpolate": "linear", - "times": [], - "addTimeMarker": false, - "defaultYExtents": false, - "setYExtents": false, - "yAxis": {}, - "mode": "stacked" - }, - "aggs": [ - {% for agg in aggs %} - { - "id": {{agg.id }}, - "type": "avg", - "schema": "metric", - "params": { - "field": "{{agg.field}}" - } - }, - {% endfor %} - { - "id": {{ aggs|length + 1 }}, - "type": "date_histogram", - "schema": "segment", - "params": { - "field": "start_date", - "interval": "auto", - "customInterval": "2h", - "min_doc_count": 1, - "extended_bounds": {} - } - } - ] -} diff --git a/dashboard/dashboard/elastic2kibana/templates/qpi.json b/dashboard/dashboard/elastic2kibana/templates/qpi.json deleted file mode 100644 index a46f315..0000000 --- a/dashboard/dashboard/elastic2kibana/templates/qpi.json +++ /dev/null @@ -1,45 +0,0 @@ -{% set aggs = aggs|default([]) -%} - -{ - "title": "qpi", - "type": "line", - "listeners": {}, - "params": { - "addLegend": true, - "shareYAxis": true, - "addTooltip": true, - "smoothLines": false, - "scale": "linear", - "interpolate": "linear", - "times": [], - "addTimeMarker": false, - "defaultYExtents": false, - "setYExtents": false, - "yAxis": {}, - "mode": "stacked" - }, - "aggs": [ - {% for agg in aggs %} - { - "id": {{agg.id }}, - "type": "avg", - "schema": "metric", - "params": { - "field": "{{agg.field}}" - } - }, - {% endfor %} - { - "id": {{ aggs|length + 1 }}, - "type": "date_histogram", - "schema": "segment", - "params": { - "field": "start_date", - "interval": "auto", - "customInterval": "2h", - "min_doc_count": 1, - "extended_bounds": {} - } - } - ] -} diff --git a/dashboard/dashboard/elastic2kibana/templates/success_percentage.json b/dashboard/dashboard/elastic2kibana/templates/success_percentage.json deleted file mode 100644 index 9930708..0000000 --- a/dashboard/dashboard/elastic2kibana/templates/success_percentage.json +++ /dev/null @@ -1,45 +0,0 @@ -{% set aggs = aggs|default([]) -%} - -{ - "title": "success_percentage", - "type": "line", - "listeners": {}, - "params": { - "addLegend": true, - "shareYAxis": true, - "addTooltip": true, - "smoothLines": false, - "scale": "linear", - "interpolate": "linear", - "times": [], - "addTimeMarker": false, - "defaultYExtents": false, - "setYExtents": false, - "yAxis": {}, - "mode": "stacked" - }, - "aggs": [ - {% for agg in aggs %} - { - "id": {{agg.id }}, - "type": "avg", - "schema": "metric", - "params": { - "field": "{{agg.field}}" - } - }, - {% endfor %} - { - "id": {{ aggs|length + 1 }}, - "type": "date_histogram", - "schema": "segment", - "params": { - "field": "start_date", - "interval": "auto", - "customInterval": "2h", - "min_doc_count": 1, - "extended_bounds": {} - } - } - ] -} diff --git a/dashboard/dashboard/elastic2kibana/templates/tests_failures.json b/dashboard/dashboard/elastic2kibana/templates/tests_failures.json deleted file mode 100644 index 01f9ba8..0000000 --- a/dashboard/dashboard/elastic2kibana/templates/tests_failures.json +++ /dev/null @@ -1,45 +0,0 @@ -{% set aggs = aggs|default([]) -%} - -{ - "title": "tests_failures", - "type": "histogram", - "listeners": {}, - "params": { - "addLegend": true, - "shareYAxis": true, - "addTooltip": true, - "smoothLines": false, - "scale": "linear", - "interpolate": "linear", - "times": [], - "addTimeMarker": false, - "defaultYExtents": false, - "setYExtents": false, - "yAxis": {}, - "mode": "grouped" - }, - "aggs": [ - {% for agg in aggs %} - { - "id": {{agg.id }}, - "type": "sum", - "schema": "metric", - "params": { - "field": "{{agg.field}}" - } - }, - {% endfor %} - { - "id": {{ aggs|length + 1 }}, - "type": "date_histogram", - "schema": "segment", - "params": { - "field": "start_date", - "interval": "auto", - "customInterval": "2h", - "min_doc_count": 1, - "extended_bounds": {} - } - } - ] -} diff --git a/dashboard/dashboard/elastic2kibana/templates/visualization.json b/dashboard/dashboard/elastic2kibana/templates/visualization.json deleted file mode 100644 index d51d417..0000000 --- a/dashboard/dashboard/elastic2kibana/templates/visualization.json +++ /dev/null @@ -1,32 +0,0 @@ -{% set vis = vis|default({}) -%} - - -{ - "description": "Kibana visualization for {{ vis.filters }}", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - {% for key, value in vis.filters.iteritems() if key != "metric" %} - {% if not (key == "pod_name" and value == "all") %} - { - "match": { - "{{ key }}": { - "query": "{{ value }}", - "type": "phrase" - } - } - } - {% if not loop.last %} - , - {% endif %} - {% endif %} - {% endfor %} - ] - } - }, - "scenario": 1, - "title": "{{vis.filters.project_name}} {{vis.filters.case_name}} {{vis.filters.installer}} {{vis.filters.metric}} {{vis.filters.pod_name}} {{vis.filters.scenario}}", - "uiStateJSON": {}, - "visState": {{ vis.visState }} -} - diff --git a/dashboard/dashboard/elastic2kibana/utility.py b/dashboard/dashboard/elastic2kibana/utility.py deleted file mode 100644 index 40d9202..0000000 --- a/dashboard/dashboard/elastic2kibana/utility.py +++ /dev/null @@ -1,16 +0,0 @@ -import json - -from jinja2 import Environment, PackageLoader - -env = Environment(loader=PackageLoader('dashboard', - 'elastic2kibana/templates')) -env.filters['jsonify'] = json.dumps - - -def dumps(a_dict, items): - for key in items: - a_dict[key] = json.dumps(a_dict[key]) - - -def dumps_2depth(a_dict, key1, key2): - a_dict[key1][key2] = json.dumps(a_dict[key1][key2]) diff --git a/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/dashboard/dashboard/elastic2kibana/visualization_assembler.py deleted file mode 100644 index d7e6e54..0000000 --- a/dashboard/dashboard/elastic2kibana/visualization_assembler.py +++ /dev/null @@ -1,84 +0,0 @@ -import json - -import utility -from dashboard.common import elastic_access - - -class VisStateBuilder(object): - def __init__(self, vis_p): - super(VisStateBuilder, self).__init__() - self.vis_p = vis_p - - def build(self): - name = self.vis_p.get('name') - fields = self.vis_p.get('fields') - - aggs = [] - index = 1 - for field in fields: - aggs.append({ - "id": index, - "field": field.get("field") - }) - index += 1 - - template = utility.env.get_template('{}.json'.format(name)) - vis = template.render(aggs=aggs) - return json.loads(vis) - - -class VisualizationAssembler(object): - def __init__(self, - project, - case, - installer, - pod, - scenario, - vis_p, - es_url, - es_creds): - super(VisualizationAssembler, self).__init__() - self.project = project - self.case = case - self.installer = installer - self.pod = pod - self.scenario = scenario - self.vis_p = vis_p - self.es_url = es_url - self.es_creds = es_creds - self._assemble() - self._publish() - - def _assemble(self): - visState = VisStateBuilder(self.vis_p).build() - self.vis_state_title = visState['title'] - - vis = { - "visState": json.dumps(visState), - "filters": { - "project_name": self.project, - "case_name": self.case, - "installer": self.installer, - "metric": self.vis_state_title, - "pod_name": self.pod, - "scenario": self.scenario - } - } - - template = utility.env.get_template('visualization.json') - - self.visualization = json.loads(template.render(vis=vis)) - utility.dumps(self.visualization, - ['visState', 'description', 'uiStateJSON']) - utility.dumps_2depth(self.visualization, - 'kibanaSavedObjectMeta', - 'searchSourceJSON') - title = self.visualization['title'] - self.id = title.replace(' ', '-').replace('/', '-') - - def _publish(self): - elastic_access.publish_kibana(self.es_url, - self.es_creds, - 'visualization', - self.id, - self.visualization) |