Merge "separate visualization building from dashboard building"

7 files changed, 225 insertions, 173 deletions

diff --git a/utils/test/dashboard/dashboard/common/elastic_access.py b/utils/test/dashboard/dashboard/common/elastic_access.py
index 8c6494d39..aaf776f7a 100644
--- a/utils/test/dashboard/dashboard/common/elastic_access.py
+++ b/utils/test/dashboard/dashboard/common/elastic_access.py
@@ -1,4 +1,5 @@
 import json
+import urlparse
 
 import urllib3
 
@@ -43,3 +44,8 @@ def get_docs(url, creds=None, body=None, field='_source'):
     for hit in res_data['hits']['hits']:
         docs.append(hit[field])
     return docs
+
+
+def publish_kibana(url, creds, type, id, body):
+    url = urlparse.urljoin(url, '/.kibana/{}/{}'.format(type, id))
+    publish_docs(url, creds, body)
diff --git a/utils/test/dashboard/dashboard/conf/config.py b/utils/test/dashboard/dashboard/conf/config.py
index b868999a2..143b1939a 100644
--- a/utils/test/dashboard/dashboard/conf/config.py
+++ b/utils/test/dashboard/dashboard/conf/config.py
@@ -23,8 +23,8 @@ class APIConfig:
 
     def __init__(self):
         self._default_config_location = "../etc/config.ini"
-        self.elastic_url = 'http://localhost:9200'
-        self.elastic_creds = None
+        self.es_url = 'http://localhost:9200'
+        self.es_creds = None
         self.kibana_url = None
         self.is_js = True
         self.js_path = None
@@ -64,8 +64,8 @@ class APIConfig:
             raise ParseError("%s not found" % config_location)
 
         # Linking attributes to keys from file with their sections
-        obj.elastic_url = obj._get_str_parameter("elastic", "url")
-        obj.elastic_creds = obj._get_str_parameter("elastic", "creds")
+        obj.es_url = obj._get_str_parameter("elastic", "url")
+        obj.es_creds = obj._get_str_parameter("elastic", "creds")
         obj.kibana_url = obj._get_str_parameter("kibana", "url")
         obj.is_js = obj._get_bool_parameter("kibana", "js")
         obj.js_path = obj._get_str_parameter("kibana", "js_path")
@@ -77,8 +77,8 @@ class APIConfig:
                "elastic_creds = %s \n" \
                "kibana_url = %s \n" \
                "is_js = %s \n" \
-               "js_path = %s \n" % (self.elastic_url,
-                                    self.elastic_creds,
+               "js_path = %s \n" % (self.es_url,
+                                    self.es_creds,
                                     self.kibana_url,
                                     self.is_js,
                                     self.js_path)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
new file mode 100644
index 000000000..c1e9dfb22
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
@@ -0,0 +1,59 @@
+import json
+
+import utility
+from common import elastic_access
+
+
+class DashboardAssembler(object):
+    def __init__(self,
+                 project,
+                 case,
+                 family,
+                 installer,
+                 pod,
+                 visAssemblers,
+                 es_url,
+                 es_creds):
+        super(DashboardAssembler, self).__init__()
+        self.project = project
+        self.case = case
+        self.test_family = family
+        self.installer = installer
+        self.pod = pod
+        self.visAssemblers = visAssemblers
+        self.es_url = es_url
+        self.es_creds = es_creds
+        self._assemble()
+        self._publish()
+
+    def _assemble(self):
+        db = {
+            "query": {
+                "project_name": self.project,
+                "case_name": self.case,
+                "installer": self.installer,
+                "metric": self.visAssemblers[0].vis_state_title,
+                "pod": self.pod
+            },
+            "test_family": self.test_family,
+            "ids": [visualization.id for visualization in self.visAssemblers]
+        }
+        template = utility.env.get_template('dashboard.json')
+        self.dashboard = json.loads(template.render(db=db))
+        utility.dumps(self.dashboard,
+                      ['description',
+                       'uiStateJSON',
+                       'panelsJSON',
+                       'optionsJSON'])
+        utility.dumps_2depth(self.dashboard,
+                             'kibanaSavedObjectMeta',
+                             'searchSourceJSON')
+        self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
+        return self
+
+    def _publish(self):
+        elastic_access.publish_kibana(self.es_url,
+                                      self.es_creds,
+                                      'dashboard',
+                                      self.id,
+                                      self.dashboard)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py
index ae5cbe8fa..8be0a01dd 100644
--- a/utils/test/dashboard/dashboard/elastic2kibana/main.py
+++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py
@@ -3,12 +3,13 @@ import json
 import urlparse
 
 import argparse
-from jinja2 import PackageLoader, Environment
 
 from common import elastic_access
 from common import logger_utils
+from conf import config
 from conf import testcases
-from conf.config import APIConfig
+from dashboard_assembler import DashboardAssembler
+from visualization_assembler import VisualizationsAssembler
 
 logger = logger_utils.DashboardLogger('elastic2kibana').get
 
@@ -18,154 +19,10 @@ parser.add_argument("-c", "--config-file",
                     help="Config file location")
 
 args = parser.parse_args()
-CONF = APIConfig().parse(args.config_file)
-base_elastic_url = CONF.elastic_url
-generate_inputs = CONF.is_js
-input_file_path = CONF.js_path
-kibana_url = CONF.kibana_url
-es_creds = CONF.elastic_creds
+CONF = config.APIConfig().parse(args.config_file)
 
 _installers = {'fuel', 'apex', 'compass', 'joid'}
 
-env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
-env.filters['jsonify'] = json.dumps
-
-
-def dumps(self, items):
-    for key in items:
-        self.visualization[key] = json.dumps(self.visualization[key])
-
-
-def dumps_2depth(self, key1, key2):
-    self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
-
-
-class Dashboard(dict):
-    def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
-        super(Dashboard, self).__init__()
-        self.project_name = project_name
-        self.case_name = case_name
-        self.family = family
-        self.installer = installer
-        self.pod = pod
-        self.scenarios = scenarios
-        self.visualization = visualization
-        self._visualization_title = None
-        self._kibana_visualizations = []
-        self._kibana_dashboard = None
-        self._create_visualizations()
-        self._create()
-
-    def _create_visualizations(self):
-        for scenario in self.scenarios:
-            self._kibana_visualizations.append(Visualization(self.project_name,
-                                                             self.case_name,
-                                                             self.installer,
-                                                             self.pod,
-                                                             scenario,
-                                                             self.visualization))
-
-        self._visualization_title = self._kibana_visualizations[0].vis_state_title
-
-    def _publish_visualizations(self):
-        for visualization in self._kibana_visualizations:
-            url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
-            logger.debug("publishing visualization '{}'".format(url))
-            # logger.error("_publish_visualization: %s" % visualization)
-            elastic_access.publish_docs(url, es_creds, visualization)
-
-    def _create(self):
-        db = {
-            "query": {
-                "project_name": self.project_name,
-                "case_name": self.case_name,
-                "installer": self.installer,
-                "metric": self._visualization_title,
-                "pod": self.pod
-            },
-            "test_family": self.family,
-            "ids": [visualization.id for visualization in self._kibana_visualizations]
-        }
-        template = env.get_template('dashboard.json')
-        self.dashboard = json.loads(template.render(db=db))
-        dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON'])
-        dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON')
-        self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
-
-
-    def _publish(self):
-        url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
-        logger.debug("publishing dashboard '{}'".format(url))
-        #logger.error("dashboard: %s" % json.dumps(self.dashboard))
-        elastic_access.publish_docs(url, es_creds, self.dashboard)
-
-    def publish(self):
-        self._publish_visualizations()
-        self._publish()
-
-
-class VisStateBuilder(object):
-    def __init__(self, visualization):
-        super(VisStateBuilder, self).__init__()
-        self.visualization = visualization
-
-    def build(self):
-        name = self.visualization.get('name')
-        fields = self.visualization.get('fields')
-
-        aggs = []
-        index = 1
-        for field in fields:
-            aggs.append({
-                "id": index,
-                "field": field.get("field")
-            })
-            index += 1
-
-        template = env.get_template('{}.json'.format(name))
-        vis = template.render(aggs=aggs)
-        return json.loads(vis)
-
-
-class Visualization(object):
-    def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
-        """
-        We need two things
-        1. filter created from
-            project_name
-            case_name
-            installer
-            pod
-            scenario
-        2. visualization state
-            field for y axis (metric) with type (avg, sum, etc.)
-            field for x axis (segment) with type (date_histogram)
-
-        :return:
-        """
-        super(Visualization, self).__init__()
-        visState = VisStateBuilder(visualization).build()
-        self.vis_state_title = visState['title']
-
-        vis = {
-            "visState": json.dumps(visState),
-            "filters": {
-                "project_name": project_name,
-                "case_name": case_name,
-                "installer": installer,
-                "metric": self.vis_state_title,
-                "pod_name": pod,
-                "scenario": scenario
-            }
-        }
-
-        template = env.get_template('visualization.json')
-
-        self.visualization = json.loads(template.render(vis=vis))
-        dumps(self.visualization, ['visState', 'description', 'uiStateJSON'])
-        dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON')
-        self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
-
 
 def _get_pods_and_scenarios(project_name, case_name, installer):
     query_json = json.JSONEncoder().encode({
@@ -183,8 +40,8 @@ def _get_pods_and_scenarios(project_name, case_name, installer):
         }
     })
 
-    elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
-                                           es_creds,
+    elastic_data = elastic_access.get_docs(urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'),
+                                           CONF.es_creds,
                                            query_json)
 
     pods_and_scenarios = {}
@@ -213,24 +70,35 @@ def construct_dashboards():
 
     :return: list of KibanaDashboards
     """
-    kibana_dashboards = []
+    dashboards = []
     for project, case_dicts in testcases.testcases_yaml.items():
         for case in case_dicts:
             case_name = case.get('name')
-            visualizations = case.get('visualizations')
+            vis_ps = case.get('visualizations')
             family = case.get('test_family')
             for installer in _installers:
                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
-                for visualization in visualizations:
+                for vis_p in vis_ps:
                     for pod, scenarios in pods_and_scenarios.iteritems():
-                        kibana_dashboards.append(Dashboard(project,
-                                                           case_name,
-                                                           family,
-                                                           installer,
-                                                           pod,
-                                                           scenarios,
-                                                           visualization))
-    return kibana_dashboards
+                        vissAssember = VisualizationsAssembler(project,
+                                                              case_name,
+                                                              installer,
+                                                              pod,
+                                                              scenarios,
+                                                              vis_p,
+                                                              CONF.es_url,
+                                                              CONF.es_creds)
+                        dashboardAssembler = DashboardAssembler(project,
+                                                                case_name,
+                                                                family,
+                                                                installer,
+                                                                pod,
+                                                                vissAssember.visAssemblers,
+                                                                CONF.es_url,
+                                                                CONF.es_creds)
+                        dashboards.append(dashboardAssembler)
+
+    return dashboards
 
 
 def generate_js_inputs(js_file_path, kibana_url, dashboards):
@@ -264,8 +132,5 @@ def generate_js_inputs(js_file_path, kibana_url, dashboards):
 def main():
     dashboards = construct_dashboards()
 
-    for kibana_dashboard in dashboards:
-        kibana_dashboard.publish()
-
-    if generate_inputs:
-        generate_js_inputs(input_file_path, kibana_url, dashboards)
+    if CONF.is_js:
+        generate_js_inputs(CONF.js_path, CONF.kibana_url, dashboards)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/utility.py b/utils/test/dashboard/dashboard/elastic2kibana/utility.py
new file mode 100644
index 000000000..dccd28aed
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/utility.py
@@ -0,0 +1,15 @@
+import json
+
+from jinja2 import Environment, PackageLoader
+
+env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
+env.filters['jsonify'] = json.dumps
+
+
+def dumps(a_dict, items):
+    for key in items:
+        a_dict[key] = json.dumps(a_dict[key])
+
+
+def dumps_2depth(a_dict, key1, key2):
+    a_dict[key1][key2] = json.dumps(a_dict[key1][key2])
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py
new file mode 100644
index 000000000..e3b6b0d38
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py
@@ -0,0 +1,107 @@
+import json
+
+import utility
+from common import elastic_access
+
+
+class VisStateBuilder(object):
+    def __init__(self, vis_p):
+        super(VisStateBuilder, self).__init__()
+        self.vis_p = vis_p
+
+    def build(self):
+        name = self.vis_p.get('name')
+        fields = self.vis_p.get('fields')
+
+        aggs = []
+        index = 1
+        for field in fields:
+            aggs.append({
+                "id": index,
+                "field": field.get("field")
+            })
+            index += 1
+
+        template = utility.env.get_template('{}.json'.format(name))
+        vis = template.render(aggs=aggs)
+        return json.loads(vis)
+
+
+class VisualizationAssembler(object):
+    def __init__(self,
+                 project,
+                 case,
+                 installer,
+                 pod,
+                 scenario,
+                 vis_p,
+                 es_url,
+                 es_creds):
+        super(VisualizationAssembler, self).__init__()
+        self.project = project
+        self.case = case
+        self.installer = installer
+        self.pod = pod
+        self.scenario = scenario
+        self.vis_p = vis_p
+        self.es_url = es_url
+        self.es_creds = es_creds
+        self._assemble()
+        self._publish()
+
+    def _assemble(self):
+        visState = VisStateBuilder(self.vis_p).build()
+        self.vis_state_title = visState['title']
+
+        vis = {
+            "visState": json.dumps(visState),
+            "filters": {
+                "project_name": self.project,
+                "case_name": self.case,
+                "installer": self.installer,
+                "metric": self.vis_state_title,
+                "pod_name": self.pod,
+                "scenario": self.scenario
+            }
+        }
+
+        template = utility.env.get_template('visualization.json')
+
+        self.visualization = json.loads(template.render(vis=vis))
+        utility.dumps(self.visualization,
+                      ['visState', 'description', 'uiStateJSON'])
+        utility.dumps_2depth(self.visualization,
+                             'kibanaSavedObjectMeta',
+                             'searchSourceJSON')
+        title = self.visualization['title']
+        self.id = title.replace(' ', '-').replace('/', '-')
+
+    def _publish(self):
+        elastic_access.publish_kibana(self.es_url,
+                                      self.es_creds,
+                                      'visualization',
+                                      self.id,
+                                      self.visualization)
+
+
+class VisualizationsAssembler(object):
+    def __init__(self,
+                 project,
+                 case,
+                 installer,
+                 pod,
+                 scenarios,
+                 vis_p,
+                 es_url,
+                 es_creds):
+        super(VisualizationsAssembler, self).__init__()
+        self.visAssemblers = []
+        for scenario in scenarios:
+            self.visAssemblers.append(VisualizationAssembler(project,
+                                                             case,
+                                                             installer,
+                                                             pod,
+                                                             scenario,
+                                                             vis_p,
+                                                             es_url,
+                                                             es_creds))
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py
index 23d53fc5b..a526d5319 100644
--- a/utils/test/dashboard/dashboard/mongo2elastic/main.py
+++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py
@@ -235,9 +235,9 @@ class DocumentsPublisher(object):
 
 
 def main():
-    base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic')
+    base_elastic_url = urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic')
     days = args.latest_days
-    es_creds = CONF.elastic_creds
+    es_creds = CONF.es_creds
 
     for project, case_dicts in testcases.testcases_yaml.items():
         for case_dict in case_dicts: