added ELK scripts for porting data from mongo to elasticsearch and managing kibana dashboards

Change-Id: I9edbc1535f6f9c4ca4dc8b4871a04c2ed6d1969e
Signed-off-by: Juraj Linkes <jlinkes@cisco.com>

1 files changed, 824 insertions, 0 deletions

diff --git a/scripts/create_kibana_dashboards.py b/scripts/create_kibana_dashboards.py
new file mode 100644
index 0000000..252ce21
--- /dev/null
+++ b/scripts/create_kibana_dashboards.py
@@ -0,0 +1,824 @@
+#! /usr/bin/env python
+import logging
+import argparse
+import shared_utils
+import json
+import urlparse
+
+logger = logging.getLogger('create_kibana_dashboards')
+logger.setLevel(logging.DEBUG)
+file_handler = logging.FileHandler('/var/log/{}.log'.format(__name__))
+file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
+logger.addHandler(file_handler)
+
+_installers = {'fuel', 'apex', 'compass', 'joid'}
+
+# see class VisualizationState for details on format
+_testcases = [
+    ('functest', 'Tempest',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "Tempest duration",
+                 "test_family": "VIM"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.tests"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.failures"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "Tempest nr of tests/failures",
+                 "test_family": "VIM"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.success_percentage"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "Tempest success percentage",
+                 "test_family": "VIM"
+             }
+         }
+     ]
+     ),
+
+    ('functest', 'Rally',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "Rally duration",
+                 "test_family": "VIM"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.tests"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "Rally nr of tests",
+                 "test_family": "VIM"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.success_percentage"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "Rally success percentage",
+                 "test_family": "VIM"
+             }
+         }
+     ]
+     ),
+
+    ('functest', 'vPing',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "vPing duration",
+                 "test_family": "VIM"
+             }
+         }
+     ]
+     ),
+
+    ('functest', 'vPing_userdata',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "vPing_userdata duration",
+                 "test_family": "VIM"
+             }
+         }
+     ]
+     ),
+
+    ('functest', 'ODL',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.tests"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.failures"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "ODL nr of tests/failures",
+                 "test_family": "Controller"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.success_percentage"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "ODL success percentage",
+                 "test_family": "Controller"
+             }
+         }
+     ]
+     ),
+
+    ('functest', 'ONOS',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.FUNCvirNet.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "ONOS FUNCvirNet duration",
+                 "test_family": "Controller"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.FUNCvirNet.tests"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.FUNCvirNet.failures"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "ONOS FUNCvirNet nr of tests/failures",
+                 "test_family": "Controller"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.FUNCvirNetL3.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "ONOS FUNCvirNetL3 duration",
+                 "test_family": "Controller"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.FUNCvirNetL3.tests"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.FUNCvirNetL3.failures"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "ONOS FUNCvirNetL3 nr of tests/failures",
+                 "test_family": "Controller"
+             }
+         }
+     ]
+     ),
+
+    ('functest', 'vIMS',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.sig_test.tests"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.sig_test.failures"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.sig_test.passed"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.sig_test.skipped"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "vIMS nr of tests/failures/passed/skipped",
+                 "test_family": "Features"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.vIMS.duration"
+                     }
+                 },
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.orchestrator.duration"
+                     }
+                 },
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.sig_test.duration"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "vIMS/ochestrator/test duration",
+                 "test_family": "Features"
+             }
+         }
+     ]
+     ),
+
+    ('promise', 'promise',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "promise duration",
+                 "test_family": "Features"
+             }
+         },
+
+         {
+             "metrics": [
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.tests"
+                     }
+                 },
+                 {
+                     "type": "sum",
+                     "params": {
+                         "field": "details.failures"
+                     }
+                 }
+             ],
+             "type": "histogram",
+             "metadata": {
+                 "label": "promise nr of tests/failures",
+                 "test_family": "Features"
+             }
+         }
+     ]
+     ),
+
+    ('doctor', 'doctor-notification',
+     [
+         {
+             "metrics": [
+                 {
+                     "type": "avg",
+                     "params": {
+                         "field": "details.duration"
+                     }
+                 }
+             ],
+             "type": "line",
+             "metadata": {
+                 "label": "doctor-notification duration",
+                 "test_family": "Features"
+             }
+         }
+     ]
+     )
+]
+
+
+class KibanaDashboard(dict):
+    def __init__(self, project_name, case_name, installer, pod, versions, visualization_detail):
+        super(KibanaDashboard, self).__init__()
+        self.project_name = project_name
+        self.case_name = case_name
+        self.installer = installer
+        self.pod = pod
+        self.versions = versions
+        self.visualization_detail = visualization_detail
+        self._visualization_title = None
+        self._kibana_visualizations = []
+        self._kibana_dashboard = None
+        self._create_visualizations()
+        self._create()
+
+    def _create_visualizations(self):
+        for version in self.versions:
+            self._kibana_visualizations.append(KibanaVisualization(self.project_name,
+                                                                   self.case_name,
+                                                                   self.installer,
+                                                                   self.pod,
+                                                                   version,
+                                                                   self.visualization_detail))
+
+        self._visualization_title = self._kibana_visualizations[0].vis_state_title
+
+    def _publish_visualizations(self):
+        for visualization in self._kibana_visualizations:
+            url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
+            logger.debug("publishing visualization '{}'".format(url))
+            shared_utils.publish_json(visualization, es_user, es_passwd, url)
+
+    def _construct_panels(self):
+        size_x = 6
+        size_y = 3
+        max_columns = 7
+        column = 1
+        row = 1
+        panel_index = 1
+        panels_json = []
+        for visualization in self._kibana_visualizations:
+            panels_json.append({
+                "id": visualization.id,
+                "type": 'visualization',
+                "panelIndex": panel_index,
+                "size_x": size_x,
+                "size_y": size_y,
+                "col": column,
+                "row": row
+            })
+            panel_index += 1
+            column += size_x
+            if column > max_columns:
+                column = 1
+                row += size_y
+        return json.dumps(panels_json, separators=(',', ':'))
+
+    def _create(self):
+        self['title'] = '{} {} {} {} {}'.format(self.project_name,
+                                                self.case_name,
+                                                self.installer,
+                                                self._visualization_title,
+                                                self.pod)
+        self.id = self['title'].replace(' ', '-').replace('/', '-')
+
+        self['hits'] = 0
+        self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
+                              " pod '{}'".format(self.project_name,
+                                                 self.case_name,
+                                                 self.installer,
+                                                 self._visualization_title,
+                                                 self.pod)
+        self['panelsJSON'] = self._construct_panels()
+        self['optionsJSON'] = json.dumps({
+            "darkTheme": False
+        },
+            separators=(',', ':'))
+        self['uiStateJSON'] = "{}"
+        self['version'] = 1
+        self['timeRestore'] = False
+        self['kibanaSavedObjectMeta'] = {
+            'searchSourceJSON': json.dumps({
+                "filter": [
+                    {
+                        "query": {
+                            "query_string": {
+                                "query": "*",
+                                "analyze_wildcard": True
+                            }
+                        }
+                    }
+                ]
+            },
+                separators=(',', ':'))
+        }
+        self['metadata'] = self.visualization_detail['metadata']
+
+    def _publish(self):
+        url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
+        logger.debug("publishing dashboard '{}'".format(url))
+        shared_utils.publish_json(self, es_user, es_passwd, url)
+
+    def publish(self):
+        self._publish_visualizations()
+        self._publish()
+
+
+class KibanaSearchSourceJSON(dict):
+    """
+    "filter": [
+                    {"match": {"installer": {"query": installer, "type": "phrase"}}},
+                    {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+                    {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
+                ]
+    """
+
+    def __init__(self, project_name, case_name, installer, pod, version):
+        super(KibanaSearchSourceJSON, self).__init__()
+        self["filter"] = [
+            {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+            {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
+            {"match": {"installer": {"query": installer, "type": "phrase"}}},
+            {"match": {"version": {"query": version, "type": "phrase"}}}
+        ]
+        if pod != 'all':
+            self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
+
+
+class VisualizationState(dict):
+    def __init__(self, input_dict):
+        """
+        dict structure:
+            {
+            "metrics":
+                [
+                    {
+                        "type": type,           # default sum
+                        "params": {
+                            "field": field      # mandatory, no default
+                    },
+                    {metric2}
+                ],
+            "segments":
+                [
+                    {
+                        "type": type,           # default date_histogram
+                        "params": {
+                            "field": field      # default creation_date
+                    },
+                    {segment2}
+                ],
+            "type": type,                       # default area
+            "mode": mode,                       # default grouped for type 'histogram', stacked for other types
+            "metadata": {
+                    "label": "Tempest duration",# mandatory, no default
+                    "test_family": "VIM"        # mandatory, no default
+                }
+            }
+
+        default modes:
+            type histogram: grouped
+            type area: stacked
+
+        :param input_dict:
+        :return:
+        """
+        super(VisualizationState, self).__init__()
+        metrics = input_dict['metrics']
+        segments = [] if 'segments' not in input_dict else input_dict['segments']
+
+        graph_type = 'area' if 'type' not in input_dict else input_dict['type']
+        self['type'] = graph_type
+
+        if 'mode' not in input_dict:
+            if graph_type == 'histogram':
+                mode = 'grouped'
+            else:
+                # default
+                mode = 'stacked'
+        else:
+            mode = input_dict['mode']
+        self['params'] = {
+            "shareYAxis": True,
+            "addTooltip": True,
+            "addLegend": True,
+            "smoothLines": False,
+            "scale": "linear",
+            "interpolate": "linear",
+            "mode": mode,
+            "times": [],
+            "addTimeMarker": False,
+            "defaultYExtents": False,
+            "setYExtents": False,
+            "yAxis": {}
+        }
+
+        self['aggs'] = []
+
+        i = 1
+        for metric in metrics:
+            self['aggs'].append({
+                "id": str(i),
+                "type": 'sum' if 'type' not in metric else metric['type'],
+                "schema": "metric",
+                "params": {
+                    "field": metric['params']['field']
+                }
+            })
+            i += 1
+
+        if len(segments) > 0:
+            for segment in segments:
+                self['aggs'].append({
+                    "id": str(i),
+                    "type": 'date_histogram' if 'type' not in segment else segment['type'],
+                    "schema": "metric",
+                    "params": {
+                        "field": "creation_date" if ('params' not in segment or 'field' not in segment['params'])
+                        else segment['params']['field'],
+                        "interval": "auto",
+                        "customInterval": "2h",
+                        "min_doc_count": 1,
+                        "extended_bounds": {}
+                    }
+                })
+                i += 1
+        else:
+            self['aggs'].append({
+                "id": str(i),
+                "type": 'date_histogram',
+                "schema": "segment",
+                "params": {
+                    "field": "creation_date",
+                    "interval": "auto",
+                    "customInterval": "2h",
+                    "min_doc_count": 1,
+                    "extended_bounds": {}
+                }
+            })
+
+        self['listeners'] = {}
+        self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
+                                  if x['schema'] == 'metric'])
+
+
+class KibanaVisualization(dict):
+    def __init__(self, project_name, case_name, installer, pod, version, detail):
+        """
+        We need two things
+        1. filter created from
+            project_name
+            case_name
+            installer
+            pod
+            version
+        2. visualization state
+            field for y axis (metric) with type (avg, sum, etc.)
+            field for x axis (segment) with type (date_histogram)
+
+        :return:
+        """
+        super(KibanaVisualization, self).__init__()
+        vis_state = VisualizationState(detail)
+        self.vis_state_title = vis_state['title']
+        self['title'] = '{} {} {} {} {} {}'.format(project_name,
+                                                   case_name,
+                                                   self.vis_state_title,
+                                                   installer,
+                                                   pod,
+                                                   version)
+        self.id = self['title'].replace(' ', '-').replace('/', '-')
+        self['visState'] = json.dumps(vis_state, separators=(',', ':'))
+        self['uiStateJSON'] = "{}"
+        self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
+                              " pod '{}' and version '{}'".format(project_name,
+                                                                  case_name,
+                                                                  self.vis_state_title,
+                                                                  installer,
+                                                                  pod,
+                                                                  version)
+        self['version'] = 1
+        self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
+                                                                                               case_name,
+                                                                                               installer,
+                                                                                               pod,
+                                                                                               version),
+                                                                        separators=(',', ':'))}
+
+
+def _get_pods_and_versions(project_name, case_name, installer):
+    query_json = json.JSONEncoder().encode({
+        "query": {
+            "bool": {
+                "must": [
+                    {"match_all": {}}
+                ],
+                "filter": [
+                    {"match": {"installer": {"query": installer, "type": "phrase"}}},
+                    {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+                    {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
+                ]
+            }
+        }
+    })
+
+    elastic_data = shared_utils.get_elastic_data(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
+                                                 es_user, es_passwd, query_json)
+
+    pods_and_versions = {}
+
+    for data in elastic_data:
+        pod = data['pod_name']
+        if pod in pods_and_versions:
+            pods_and_versions[pod].add(data['version'])
+        else:
+            pods_and_versions[pod] = {data['version']}
+
+        if 'all' in pods_and_versions:
+            pods_and_versions['all'].add(data['version'])
+        else:
+            pods_and_versions['all'] = {data['version']}
+
+    return pods_and_versions
+
+
+def construct_dashboards():
+    """
+    iterate over testcase and installer
+    1. get available pods for each testcase/installer pair
+    2. get available version for each testcase/installer/pod tuple
+    3. construct KibanaInput and append
+
+    :return: list of KibanaDashboards
+    """
+    kibana_dashboards = []
+    for project_name, case_name, visualization_details in _testcases:
+        for installer in _installers:
+            pods_and_versions = _get_pods_and_versions(project_name, case_name, installer)
+            for visualization_detail in visualization_details:
+                for pod, versions in pods_and_versions.iteritems():
+                    kibana_dashboards.append(KibanaDashboard(project_name, case_name, installer, pod, versions,
+                                                             visualization_detail))
+    return kibana_dashboards
+
+
+def generate_js_inputs(js_file_path, kibana_url, dashboards):
+    js_dict = {}
+    for dashboard in dashboards:
+        dashboard_meta = dashboard['metadata']
+        test_family = dashboard_meta['test_family']
+        test_label = dashboard_meta['label']
+
+        if test_family not in js_dict:
+            js_dict[test_family] = {}
+
+        js_test_family = js_dict[test_family]
+
+        if test_label not in js_test_family:
+            js_test_family[test_label] = {}
+
+        js_test_label = js_test_family[test_label]
+
+        if dashboard.installer not in js_test_label:
+            js_test_label[dashboard.installer] = {}
+
+        js_installer = js_test_label[dashboard.installer]
+        js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
+
+    with open(js_file_path, 'w+') as js_file_fdesc:
+        js_file_fdesc.write('var kibana_dashboard_links = ')
+        js_file_fdesc.write(str(js_dict).replace("u'", "'"))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
+    parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
+                        help='the url of elasticsearch, defaults to http://localhost:9200')
+    parser.add_argument('-js', '--generate_js_inputs', action='store_true',
+                        help='Use this argument to generate javascript inputs for kibana landing page')
+    parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
+                        help='Path of javascript file with inputs for kibana landing page')
+    parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
+                        help='The url of kibana for javascript inputs')
+
+    parser.add_argument('-u', '--elasticsearch-username',
+                        help='the username for elasticsearch')
+
+    parser.add_argument('-p', '--elasticsearch-password',
+                        help='the password for elasticsearch')
+
+    args = parser.parse_args()
+    base_elastic_url = args.elasticsearch_url
+    generate_inputs = args.generate_js_inputs
+    input_file_path = args.js_path
+    kibana_url = args.kibana_url
+    es_user = args.elasticsearch_username
+    es_passwd = args.elasticsearch_password
+
+    dashboards = construct_dashboards()
+
+    for kibana_dashboard in dashboards:
+        kibana_dashboard.publish()
+
+    if generate_inputs:
+        generate_js_inputs(input_file_path, kibana_url, dashboards)