From e7b7bb64bf2f3f03980504d80f304e4859146ed2 Mon Sep 17 00:00:00 2001
From: SerenaFeng <feng.xiaowei@zte.com.cn>
Date: Thu, 22 Sep 2016 16:15:58 +0800
Subject: rebuild directory structure of Kibana dashboard
JIRA: FUNCTEST-465
Change-Id: Icecd350b2f67105c8aaa9d71fd76d24827515545
Signed-off-by: SerenaFeng <feng.xiaowei@zte.com.cn>
---
utils/test/dashboard.tar.gz | Bin 0 -> 14627 bytes
utils/test/dashboard/backup-db.sh | 33 ++
utils/test/dashboard/dashboard/__init__.py | 0
utils/test/dashboard/dashboard/common/__init__.py | 0
.../dashboard/dashboard/common/elastic_access.py | 64 ++++
.../dashboard/dashboard/common/logger_utils.py | 65 ++++
utils/test/dashboard/dashboard/conf/__init__.py | 0
utils/test/dashboard/dashboard/conf/config.py | 88 +++++
utils/test/dashboard/dashboard/conf/testcases.py | 24 ++
.../dashboard/dashboard/elastic2kibana/__init__.py | 0
.../dashboard/dashboard/elastic2kibana/main.py | 373 ++++++++++++++++++++
.../dashboard/dashboard/elastic2kibana_main.py | 4 +
.../test/dashboard/dashboard/functest/__init__.py | 0
.../dashboard/dashboard/functest/testcases.yaml | 138 ++++++++
.../dashboard/dashboard/mongo2elastic/__init__.py | 0
.../dashboard/dashboard/mongo2elastic/format.py | 186 ++++++++++
.../test/dashboard/dashboard/mongo2elastic/main.py | 243 +++++++++++++
.../test/dashboard/dashboard/mongo2elastic_main.py | 4 +
utils/test/dashboard/etc/config.ini | 14 +
utils/test/dashboard/kibana_cleanup.py | 41 +++
utils/test/scripts/backup-db.sh | 33 --
utils/test/scripts/config.ini | 14 -
utils/test/scripts/config.py | 88 -----
utils/test/scripts/create_kibana_dashboards.py | 375 ---------------------
utils/test/scripts/kibana_cleanup.py | 41 ---
utils/test/scripts/logger_utils.py | 65 ----
utils/test/scripts/mongo2elastic_format.py | 186 ----------
utils/test/scripts/mongo_to_elasticsearch.py | 248 --------------
utils/test/scripts/shared_utils.py | 64 ----
utils/test/scripts/testcases.yaml | 138 --------
utils/test/scripts/testcases_parser.py | 24 --
31 files changed, 1277 insertions(+), 1276 deletions(-)
create mode 100644 utils/test/dashboard.tar.gz
create mode 100644 utils/test/dashboard/backup-db.sh
create mode 100644 utils/test/dashboard/dashboard/__init__.py
create mode 100644 utils/test/dashboard/dashboard/common/__init__.py
create mode 100644 utils/test/dashboard/dashboard/common/elastic_access.py
create mode 100644 utils/test/dashboard/dashboard/common/logger_utils.py
create mode 100644 utils/test/dashboard/dashboard/conf/__init__.py
create mode 100644 utils/test/dashboard/dashboard/conf/config.py
create mode 100644 utils/test/dashboard/dashboard/conf/testcases.py
create mode 100644 utils/test/dashboard/dashboard/elastic2kibana/__init__.py
create mode 100644 utils/test/dashboard/dashboard/elastic2kibana/main.py
create mode 100644 utils/test/dashboard/dashboard/elastic2kibana_main.py
create mode 100644 utils/test/dashboard/dashboard/functest/__init__.py
create mode 100644 utils/test/dashboard/dashboard/functest/testcases.yaml
create mode 100644 utils/test/dashboard/dashboard/mongo2elastic/__init__.py
create mode 100644 utils/test/dashboard/dashboard/mongo2elastic/format.py
create mode 100644 utils/test/dashboard/dashboard/mongo2elastic/main.py
create mode 100644 utils/test/dashboard/dashboard/mongo2elastic_main.py
create mode 100644 utils/test/dashboard/etc/config.ini
create mode 100644 utils/test/dashboard/kibana_cleanup.py
delete mode 100644 utils/test/scripts/backup-db.sh
delete mode 100644 utils/test/scripts/config.ini
delete mode 100644 utils/test/scripts/config.py
delete mode 100644 utils/test/scripts/create_kibana_dashboards.py
delete mode 100644 utils/test/scripts/kibana_cleanup.py
delete mode 100644 utils/test/scripts/logger_utils.py
delete mode 100644 utils/test/scripts/mongo2elastic_format.py
delete mode 100644 utils/test/scripts/mongo_to_elasticsearch.py
delete mode 100644 utils/test/scripts/shared_utils.py
delete mode 100644 utils/test/scripts/testcases.yaml
delete mode 100644 utils/test/scripts/testcases_parser.py
diff --git a/utils/test/dashboard.tar.gz b/utils/test/dashboard.tar.gz
new file mode 100644
index 000000000..ef85f90da
Binary files /dev/null and b/utils/test/dashboard.tar.gz differ
diff --git a/utils/test/dashboard/backup-db.sh b/utils/test/dashboard/backup-db.sh
new file mode 100644
index 000000000..35c3fbe5a
--- /dev/null
+++ b/utils/test/dashboard/backup-db.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2016 Orange and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+echo "Backup Test collection DB"
+now=$(date +"%m_%d_%Y_%H_%M_%S")
+echo $now
+echo " ------------- "
+TARGET_DIR=./$now
+TEST_RESULT_DB_BACKUP="test_collection_db."$now".tar.gz"
+
+echo "Create Directory for backup"
+mkdir -p $TARGET_DIR
+
+echo "Export results"
+mongoexport --db test_results_collection -c results --out $TARGET_DIR/backup-results.json
+echo "Export test cases"
+mongoexport --db test_results_collection -c testcases --out $TARGET_DIR/backup-cases.json
+echo "Export projects"
+mongoexport --db test_results_collection -c projects --out $TARGET_DIR/backup-projects.json
+echo "Export pods"
+mongoexport --db test_results_collection -c pods --out $TARGET_DIR/backup-pod.json
+
+echo "Create tar.gz"
+#tar -cvzf $TEST_RESULT_DB_BACKUP $TARGET_DIR
+
+echo "Delete temp directory"
+#rm -Rf $TARGET_DIR
diff --git a/utils/test/dashboard/dashboard/__init__.py b/utils/test/dashboard/dashboard/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/test/dashboard/dashboard/common/__init__.py b/utils/test/dashboard/dashboard/common/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/test/dashboard/dashboard/common/elastic_access.py b/utils/test/dashboard/dashboard/common/elastic_access.py
new file mode 100644
index 000000000..e90a17fa3
--- /dev/null
+++ b/utils/test/dashboard/dashboard/common/elastic_access.py
@@ -0,0 +1,64 @@
+import json
+
+import urllib3
+
+http = urllib3.PoolManager()
+
+
+def delete_request(url, creds, body=None):
+ headers = urllib3.make_headers(basic_auth=creds)
+ http.request('DELETE', url, headers=headers, body=body)
+
+
+def publish_json(json_ojb, creds, to):
+ json_dump = json.dumps(json_ojb)
+ if to == 'stdout':
+ print json_dump
+ return 200, None
+ else:
+ headers = urllib3.make_headers(basic_auth=creds)
+ result = http.request('POST', to, headers=headers, body=json_dump)
+ return result.status, result.data
+
+
+def _get_nr_of_hits(elastic_json):
+ return elastic_json['hits']['total']
+
+
+def get_elastic_docs(elastic_url, creds, body=None, field = '_source'):
+
+ # 1. get the number of results
+ headers = urllib3.make_headers(basic_auth=creds)
+ elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size=0', headers=headers, body=body).data)
+ print elastic_json
+ nr_of_hits = _get_nr_of_hits(elastic_json)
+
+ # 2. get all results
+ elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size={}'.format(nr_of_hits), headers=headers, body=body).data)
+
+ elastic_docs = []
+ for hit in elastic_json['hits']['hits']:
+ elastic_docs.append(hit[field])
+ return elastic_docs
+
+
+def get_elastic_docs_by_days(elastic_url, creds, days):
+ if days == 0:
+ body = '''{
+ "query": {
+ "match_all": {}
+ }
+ }'''
+ elif days > 0:
+ body = '''{{
+ "query" : {{
+ "range" : {{
+ "start_date" : {{
+ "gte" : "now-{}d"
+ }}
+ }}
+ }}
+ }}'''.format(days)
+ else:
+ raise Exception('Update days must be non-negative')
+ return get_elastic_docs(elastic_url, creds, body)
diff --git a/utils/test/dashboard/dashboard/common/logger_utils.py b/utils/test/dashboard/dashboard/common/logger_utils.py
new file mode 100644
index 000000000..183080810
--- /dev/null
+++ b/utils/test/dashboard/dashboard/common/logger_utils.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+#
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Logging levels:
+# Level Numeric value
+# CRITICAL 50
+# ERROR 40
+# WARNING 30
+# INFO 20
+# DEBUG 10
+# NOTSET 0
+#
+# Usage:
+# import functest_logger as fl
+# logger = fl.Logger("script_name").getLogger()
+# logger.info("message to be shown with - INFO - ")
+# logger.debug("message to be shown with - DEBUG -")
+
+import logging
+import os
+
+
+class Logger(object):
+ file_path = '/var/log'
+ formatter = logging.Formatter('%(asctime)s - %(name)s - '
+ '%(levelname)s - %(message)s')
+
+ def __init__(self, logger_name):
+
+ IF_DEBUG = os.getenv('IF_DEBUG')
+
+ self.logger_name = logger_name
+ self.logger = logging.getLogger(logger_name)
+ self.logger.propagate = 0
+ self.logger.setLevel(logging.DEBUG)
+
+ ch = logging.StreamHandler()
+ ch.setFormatter(self.formatter)
+ if IF_DEBUG is not None and IF_DEBUG.lower() == "true":
+ ch.setLevel(logging.DEBUG)
+ else:
+ ch.setLevel(logging.INFO)
+ self.logger.addHandler(ch)
+
+ hdlr = logging.FileHandler('%s/%s.log' % (self.file_path, logger_name))
+ hdlr.setFormatter(self.formatter)
+ hdlr.setLevel(logging.DEBUG)
+ self.logger.addHandler(hdlr)
+
+ @property
+ def get(self):
+ return self.logger
+
+
+class DashboardLogger(Logger):
+ file_path = '/var/log/kibana_dashboard'
+
+ def __init__(self, logger_name):
+ super(DashboardLogger, self).__init__(logger_name)
+
diff --git a/utils/test/dashboard/dashboard/conf/__init__.py b/utils/test/dashboard/dashboard/conf/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/test/dashboard/dashboard/conf/config.py b/utils/test/dashboard/dashboard/conf/config.py
new file mode 100644
index 000000000..2e0f1cabb
--- /dev/null
+++ b/utils/test/dashboard/dashboard/conf/config.py
@@ -0,0 +1,88 @@
+#! /usr/bin/env python
+
+from ConfigParser import SafeConfigParser, NoOptionError
+
+
+class ParseError(Exception):
+ """
+ Custom exception class for config file
+ """
+
+ def __init__(self, message):
+ self.msg = message
+
+ def __str__(self):
+ return 'error parsing config file : %s' % self.msg
+
+
+class APIConfig:
+ """
+ The purpose of this class is to load values correctly from the config file.
+ Each key is declared as an attribute in __init__() and linked in parse()
+ """
+
+ def __init__(self):
+ self._default_config_location = "../etc/config.ini"
+ self.elastic_url = 'http://localhost:9200'
+ self.elastic_creds = None
+ self.destination = 'elasticsearch'
+ self.kibana_url = None
+ self.is_js = True
+ self.js_path = None
+
+ def _get_str_parameter(self, section, param):
+ try:
+ return self._parser.get(section, param)
+ except NoOptionError:
+ raise ParseError("[%s.%s] parameter not found" % (section, param))
+
+ def _get_int_parameter(self, section, param):
+ try:
+ return int(self._get_str_parameter(section, param))
+ except ValueError:
+ raise ParseError("[%s.%s] not an int" % (section, param))
+
+ def _get_bool_parameter(self, section, param):
+ result = self._get_str_parameter(section, param)
+ if str(result).lower() == 'true':
+ return True
+ if str(result).lower() == 'false':
+ return False
+
+ raise ParseError(
+ "[%s.%s : %s] not a boolean" % (section, param, result))
+
+ @staticmethod
+ def parse(config_location=None):
+ obj = APIConfig()
+
+ if config_location is None:
+ config_location = obj._default_config_location
+
+ obj._parser = SafeConfigParser()
+ obj._parser.read(config_location)
+ if not obj._parser:
+ raise ParseError("%s not found" % config_location)
+
+ # Linking attributes to keys from file with their sections
+ obj.elastic_url = obj._get_str_parameter("elastic", "url")
+ obj.elastic_creds = obj._get_str_parameter("elastic", "creds")
+ obj.destination = obj._get_str_parameter("output", "destination")
+ obj.kibana_url = obj._get_str_parameter("kibana", "url")
+ obj.is_js = obj._get_bool_parameter("kibana", "js")
+ obj.js_path = obj._get_str_parameter("kibana", "js_path")
+
+ return obj
+
+ def __str__(self):
+ return "elastic_url = %s \n" \
+ "elastic_creds = %s \n" \
+ "destination = %s \n" \
+ "kibana_url = %s \n" \
+ "is_js = %s \n" \
+ "js_path = %s \n" % (self.elastic_url,
+ self.elastic_creds,
+ self.destination,
+ self.kibana_url,
+ self.is_js,
+ self.js_path)
diff --git a/utils/test/dashboard/dashboard/conf/testcases.py b/utils/test/dashboard/dashboard/conf/testcases.py
new file mode 100644
index 000000000..e120987dd
--- /dev/null
+++ b/utils/test/dashboard/dashboard/conf/testcases.py
@@ -0,0 +1,24 @@
+import yaml
+
+
+with open('./functest/testcases.yaml') as f:
+ testcases_yaml = yaml.safe_load(f)
+f.close()
+
+
+def compose_format(fmt):
+ return 'format_' + fmt.strip()
+
+
+def get_format(project, case):
+ testcases = testcases_yaml.get(project)
+ if isinstance(testcases, list):
+ for case_dict in testcases:
+ if case_dict['name'] == case:
+ return compose_format(case_dict['format'])
+ return None
+
+
+if __name__ == '__main__':
+ fmt = get_format('functest', 'vping_ssh')
+ print fmt
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/__init__.py b/utils/test/dashboard/dashboard/elastic2kibana/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py
new file mode 100644
index 000000000..c1cbc308e
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py
@@ -0,0 +1,373 @@
+#! /usr/bin/env python
+import json
+import urlparse
+
+import argparse
+
+from common import logger_utils, elastic_access
+from conf import testcases
+from conf.config import APIConfig
+
+logger = logger_utils.DashboardLogger('elastic2kibana').get
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-c", "--config-file",
+ dest='config_file',
+ help="Config file location")
+
+args = parser.parse_args()
+CONF = APIConfig().parse(args.config_file)
+base_elastic_url = CONF.elastic_url
+generate_inputs = CONF.is_js
+input_file_path = CONF.js_path
+kibana_url = CONF.kibana_url
+es_creds = CONF.elastic_creds
+
+_installers = {'fuel', 'apex', 'compass', 'joid'}
+
+
+class KibanaDashboard(dict):
+ def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
+ super(KibanaDashboard, self).__init__()
+ self.project_name = project_name
+ self.case_name = case_name
+ self.family = family
+ self.installer = installer
+ self.pod = pod
+ self.scenarios = scenarios
+ self.visualization = visualization
+ self._visualization_title = None
+ self._kibana_visualizations = []
+ self._kibana_dashboard = None
+ self._create_visualizations()
+ self._create()
+
+ def _create_visualizations(self):
+ for scenario in self.scenarios:
+ self._kibana_visualizations.append(KibanaVisualization(self.project_name,
+ self.case_name,
+ self.installer,
+ self.pod,
+ scenario,
+ self.visualization))
+
+ self._visualization_title = self._kibana_visualizations[0].vis_state_title
+
+ def _publish_visualizations(self):
+ for visualization in self._kibana_visualizations:
+ url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
+ logger.debug("publishing visualization '{}'".format(url))
+ elastic_access.publish_json(visualization, es_creds, url)
+
+ def _construct_panels(self):
+ size_x = 6
+ size_y = 3
+ max_columns = 7
+ column = 1
+ row = 1
+ panel_index = 1
+ panels_json = []
+ for visualization in self._kibana_visualizations:
+ panels_json.append({
+ "id": visualization.id,
+ "type": 'visualization',
+ "panelIndex": panel_index,
+ "size_x": size_x,
+ "size_y": size_y,
+ "col": column,
+ "row": row
+ })
+ panel_index += 1
+ column += size_x
+ if column > max_columns:
+ column = 1
+ row += size_y
+ return json.dumps(panels_json, separators=(',', ':'))
+
+ def _create(self):
+ self['title'] = '{} {} {} {} {}'.format(self.project_name,
+ self.case_name,
+ self.installer,
+ self._visualization_title,
+ self.pod)
+ self.id = self['title'].replace(' ', '-').replace('/', '-')
+
+ self['hits'] = 0
+ self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
+ " pod '{}'".format(self.project_name,
+ self.case_name,
+ self.installer,
+ self._visualization_title,
+ self.pod)
+ self['panelsJSON'] = self._construct_panels()
+ self['optionsJSON'] = json.dumps({
+ "darkTheme": False
+ },
+ separators=(',', ':'))
+ self['uiStateJSON'] = "{}"
+ self['scenario'] = 1
+ self['timeRestore'] = False
+ self['kibanaSavedObjectMeta'] = {
+ 'searchSourceJSON': json.dumps({
+ "filter": [
+ {
+ "query": {
+ "query_string": {
+ "query": "*",
+ "analyze_wildcard": True
+ }
+ }
+ }
+ ]
+ },
+ separators=(',', ':'))
+ }
+
+ label = self.case_name
+ if 'label' in self.visualization:
+ label += " %s" % self.visualization.get('label')
+ label += " %s" % self.visualization.get('name')
+ self['metadata'] = {
+ "label": label,
+ "test_family": self.family
+ }
+
+ def _publish(self):
+ url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
+ logger.debug("publishing dashboard '{}'".format(url))
+ elastic_access.publish_json(self, es_creds, url)
+
+ def publish(self):
+ self._publish_visualizations()
+ self._publish()
+
+
+class KibanaSearchSourceJSON(dict):
+ """
+ "filter": [
+ {"match": {"installer": {"query": installer, "type": "phrase"}}},
+ {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+ {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
+ ]
+ """
+
+ def __init__(self, project_name, case_name, installer, pod, scenario):
+ super(KibanaSearchSourceJSON, self).__init__()
+ self["filter"] = [
+ {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+ {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
+ {"match": {"installer": {"query": installer, "type": "phrase"}}},
+ {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
+ ]
+ if pod != 'all':
+ self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
+
+
+class VisualizationState(dict):
+ def __init__(self, visualization):
+ super(VisualizationState, self).__init__()
+ name = visualization.get('name')
+ fields = visualization.get('fields')
+
+ if name == 'tests_failures':
+ mode = 'grouped'
+ metric_type = 'sum'
+ self['type'] = 'histogram'
+ else:
+ # duration or success_percentage
+ mode = 'stacked'
+ metric_type = 'avg'
+ self['type'] = 'line'
+
+ self['params'] = {
+ "shareYAxis": True,
+ "addTooltip": True,
+ "addLegend": True,
+ "smoothLines": False,
+ "scale": "linear",
+ "interpolate": "linear",
+ "mode": mode,
+ "times": [],
+ "addTimeMarker": False,
+ "defaultYExtents": False,
+ "setYExtents": False,
+ "yAxis": {}
+ }
+
+ self['aggs'] = []
+
+ i = 1
+ for field in fields:
+ self['aggs'].append({
+ "id": str(i),
+ "type": metric_type,
+ "schema": "metric",
+ "params": {
+ "field": field.get('field')
+ }
+ })
+ i += 1
+
+ self['aggs'].append({
+ "id": str(i),
+ "type": 'date_histogram',
+ "schema": "segment",
+ "params": {
+ "field": "start_date",
+ "interval": "auto",
+ "customInterval": "2h",
+ "min_doc_count": 1,
+ "extended_bounds": {}
+ }
+ })
+
+ self['listeners'] = {}
+ self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
+ if x['schema'] == 'metric'])
+
+
+class KibanaVisualization(dict):
+ def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
+ """
+ We need two things
+ 1. filter created from
+ project_name
+ case_name
+ installer
+ pod
+ scenario
+ 2. visualization state
+ field for y axis (metric) with type (avg, sum, etc.)
+ field for x axis (segment) with type (date_histogram)
+
+ :return:
+ """
+ super(KibanaVisualization, self).__init__()
+ vis_state = VisualizationState(visualization)
+ self.vis_state_title = vis_state['title']
+ self['title'] = '{} {} {} {} {} {}'.format(project_name,
+ case_name,
+ self.vis_state_title,
+ installer,
+ pod,
+ scenario)
+ self.id = self['title'].replace(' ', '-').replace('/', '-')
+ self['visState'] = json.dumps(vis_state, separators=(',', ':'))
+ self['uiStateJSON'] = "{}"
+ self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
+ " pod '{}' and scenario '{}'".format(project_name,
+ case_name,
+ self.vis_state_title,
+ installer,
+ pod,
+ scenario)
+ self['scenario'] = 1
+ self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
+ case_name,
+ installer,
+ pod,
+ scenario),
+ separators=(',', ':'))}
+
+
+def _get_pods_and_scenarios(project_name, case_name, installer):
+ query_json = json.JSONEncoder().encode({
+ "query": {
+ "bool": {
+ "must": [
+ {"match_all": {}}
+ ],
+ "filter": [
+ {"match": {"installer": {"query": installer, "type": "phrase"}}},
+ {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+ {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
+ ]
+ }
+ }
+ })
+
+ elastic_data = elastic_access.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
+ es_creds, query_json)
+
+ pods_and_scenarios = {}
+
+ for data in elastic_data:
+ pod = data['pod_name']
+ if pod in pods_and_scenarios:
+ pods_and_scenarios[pod].add(data['scenario'])
+ else:
+ pods_and_scenarios[pod] = {data['scenario']}
+
+ if 'all' in pods_and_scenarios:
+ pods_and_scenarios['all'].add(data['scenario'])
+ else:
+ pods_and_scenarios['all'] = {data['scenario']}
+
+ return pods_and_scenarios
+
+
+def construct_dashboards():
+ """
+ iterate over testcase and installer
+ 1. get available pods for each testcase/installer pair
+ 2. get available scenario for each testcase/installer/pod tuple
+ 3. construct KibanaInput and append
+
+ :return: list of KibanaDashboards
+ """
+ kibana_dashboards = []
+ for project, case_dicts in testcases.testcases_yaml.items():
+ for case in case_dicts:
+ case_name = case.get('name')
+ visualizations = case.get('visualizations')
+ family = case.get('test_family')
+ for installer in _installers:
+ pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
+ for visualization in visualizations:
+ for pod, scenarios in pods_and_scenarios.iteritems():
+ kibana_dashboards.append(KibanaDashboard(project,
+ case_name,
+ family,
+ installer,
+ pod,
+ scenarios,
+ visualization))
+ return kibana_dashboards
+
+
+def generate_js_inputs(js_file_path, kibana_url, dashboards):
+ js_dict = {}
+ for dashboard in dashboards:
+ dashboard_meta = dashboard['metadata']
+ test_family = dashboard_meta['test_family']
+ test_label = dashboard_meta['label']
+
+ if test_family not in js_dict:
+ js_dict[test_family] = {}
+
+ js_test_family = js_dict[test_family]
+
+ if test_label not in js_test_family:
+ js_test_family[test_label] = {}
+
+ js_test_label = js_test_family[test_label]
+
+ if dashboard.installer not in js_test_label:
+ js_test_label[dashboard.installer] = {}
+
+ js_installer = js_test_label[dashboard.installer]
+ js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
+
+ with open(js_file_path, 'w+') as js_file_fdesc:
+ js_file_fdesc.write('var kibana_dashboard_links = ')
+ js_file_fdesc.write(str(js_dict).replace("u'", "'"))
+
+
+def main():
+ dashboards = construct_dashboards()
+
+ for kibana_dashboard in dashboards:
+ kibana_dashboard.publish()
+
+ if generate_inputs:
+ generate_js_inputs(input_file_path, kibana_url, dashboards)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana_main.py b/utils/test/dashboard/dashboard/elastic2kibana_main.py
new file mode 100644
index 000000000..3ec27cb40
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana_main.py
@@ -0,0 +1,4 @@
+from elastic2kibana.main import main
+
+if __name__ == '__main__':
+ main()
diff --git a/utils/test/dashboard/dashboard/functest/__init__.py b/utils/test/dashboard/dashboard/functest/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/test/dashboard/dashboard/functest/testcases.yaml b/utils/test/dashboard/dashboard/functest/testcases.yaml
new file mode 100644
index 000000000..9c33d2e6b
--- /dev/null
+++ b/utils/test/dashboard/dashboard/functest/testcases.yaml
@@ -0,0 +1,138 @@
+functest:
+ -
+ name: tempest_smoke_serial
+ format: normal
+ test_family: VIM
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.duration
+ -
+ name: tests_failures
+ fields:
+ - field: details.tests
+ - field: details.failures
+ -
+ name: success_percentage
+ fields:
+ - field: details.success_percentage
+ -
+ name: rally_sanity
+ test_family: VIM
+ format: rally
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.duration
+ -
+ name: tests_failures
+ fields:
+ - field: details.tests
+ -
+ name: success_percentage
+ fields:
+ - field: details.success_percentage
+ -
+ name: vping_ssh
+ format: normal
+ test_family: VIM
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.duration
+ -
+ name: vping_userdata
+ format: normal
+ test_family: VIM
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.duration
+ -
+ name: odl
+ test_family: Controller
+ format: odl
+ visualizations:
+ -
+ name: tests_failures
+ fields:
+ - field: details.tests
+ - field: details.failures
+ -
+ name: success_percentage
+ fields:
+ - field: details.success_percentage
+ -
+ name: onos
+ format: onos
+ test_family: Controller
+ visualizations:
+ -
+ name: duration
+ label: FUNCvirNet
+ fields:
+ - field: details.FUNCvirNet.duration
+ -
+ name: duration
+ label: FUNCvirNetL3
+ fields:
+ - field: details.FUNCvirNetL3.duration
+ -
+ name: tests_failures
+ label: FUNCvirNet
+ fields:
+ - field: details.FUNCvirNet.tests
+ - field: details.FUNCvirNet.failures
+ -
+ name: tests_failures
+ label: FUNCvirNetL3
+ fields:
+ - field: details.FUNCvirNetL3.tests
+ - field: details.FUNCvirNetL3.failures
+ -
+ name: vims
+ format: vims
+ test_family: Features
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.vIMS.duration
+ - field: details.orchestrator.duration
+ - field: details.sig_test.duration
+ -
+ name: tests_failures
+ fields:
+ - field: details.sig_test.tests
+ - field: details.sig_test.failures
+ - field: details.sig_test.passed
+ - field: details.sig_test.skipped
+promise:
+ -
+ name: promise
+ format: normal
+ test_family: Features
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.duration
+ -
+ name: tests_failures
+ fields:
+ - field: details.tests
+ - field: details.failures
+doctor:
+ -
+ name: doctor-notification
+ test_family: Features
+ format: normal
+ visualizations:
+ -
+ name: duration
+ fields:
+ - field: details.duration
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/__init__.py b/utils/test/dashboard/dashboard/mongo2elastic/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/format.py b/utils/test/dashboard/dashboard/mongo2elastic/format.py
new file mode 100644
index 000000000..ef485bae0
--- /dev/null
+++ b/utils/test/dashboard/dashboard/mongo2elastic/format.py
@@ -0,0 +1,186 @@
+#! /usr/bin/env python
+
+
+def _convert_value(value):
+ return value if value != '' else 0
+
+
+def _convert_duration(duration):
+ if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration:
+ hours, minutes, seconds = duration.split(":")
+ hours = _convert_value(hours)
+ minutes = _convert_value(minutes)
+ seconds = _convert_value(seconds)
+ int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds)
+ else:
+ int_duration = duration
+ return int_duration
+
+
+def format_normal(testcase):
+ """
+ Look for these and leave any of those:
+ details.duration
+ details.tests
+ details.failures
+
+ If none are present, then return False
+ """
+ found = False
+ testcase_details = testcase['details']
+ fields = ['duration', 'tests', 'failures']
+ if isinstance(testcase_details, dict):
+ for key, value in testcase_details.items():
+ if key in fields:
+ found = True
+ if key == 'duration':
+ testcase_details[key] = _convert_duration(value)
+ else:
+ del testcase_details[key]
+
+ if 'tests' in testcase_details and 'failures' in testcase_details:
+ testcase_tests = float(testcase_details['tests'])
+ testcase_failures = float(testcase_details['failures'])
+ if testcase_tests != 0:
+ testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests
+ else:
+ testcase_details['success_percentage'] = 0
+
+
+ return found
+
+
+def format_rally(testcase):
+ """
+ Structure:
+ details.[{summary.duration}]
+ details.[{summary.nb success}]
+ details.[{summary.nb tests}]
+
+ Find data for these fields
+ -> details.duration
+ -> details.tests
+ -> details.success_percentage
+ """
+ details = testcase['details']
+ summary = None
+ for item in details:
+ if 'summary' in item:
+ summary = item['summary']
+
+ if not summary:
+ return False
+
+ testcase['details'] = {
+ 'duration': summary['duration'],
+ 'tests': summary['nb tests'],
+ 'success_percentage': summary['nb success']
+ }
+ return True
+
+
+def _get_statistics(orig_data, stat_fields, stat_values=None):
+ test_results = {}
+ for stat_data in orig_data:
+ for field in stat_fields:
+ stat_value = stat_data[field]
+ if stat_value in test_results:
+ test_results[stat_value] += 1
+ else:
+ test_results[stat_value] = 1
+
+ if stat_values is not None:
+ for stat_value in stat_values:
+ if stat_value not in test_results:
+ test_results[stat_value] = 0
+
+ return test_results
+
+
+def format_onos(testcase):
+ """
+ Structure:
+ details.FUNCvirNet.duration
+ details.FUNCvirNet.status.[{Case result}]
+ details.FUNCvirNetL3.duration
+ details.FUNCvirNetL3.status.[{Case result}]
+
+ Find data for these fields
+ -> details.FUNCvirNet.duration
+ -> details.FUNCvirNet.tests
+ -> details.FUNCvirNet.failures
+ -> details.FUNCvirNetL3.duration
+ -> details.FUNCvirNetL3.tests
+ -> details.FUNCvirNetL3.failures
+ """
+ testcase_details = testcase['details']
+
+ if 'FUNCvirNet' not in testcase_details or 'FUNCvirNetL3' not in testcase_details:
+ return False
+
+ funcvirnet_details = testcase_details['FUNCvirNet']['status']
+ funcvirnet_stats = _get_statistics(funcvirnet_details, ('Case result',), ('PASS', 'FAIL'))
+ funcvirnet_passed = funcvirnet_stats['PASS']
+ funcvirnet_failed = funcvirnet_stats['FAIL']
+ funcvirnet_all = funcvirnet_passed + funcvirnet_failed
+
+ funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status']
+ funcvirnetl3_stats = _get_statistics(funcvirnetl3_details, ('Case result',), ('PASS', 'FAIL'))
+ funcvirnetl3_passed = funcvirnetl3_stats['PASS']
+ funcvirnetl3_failed = funcvirnetl3_stats['FAIL']
+ funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed
+
+ testcase_details['FUNCvirNet'] = {
+ 'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']),
+ 'tests': funcvirnet_all,
+ 'failures': funcvirnet_failed
+ }
+ testcase_details['FUNCvirNetL3'] = {
+ 'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']),
+ 'tests': funcvirnetl3_all,
+ 'failures': funcvirnetl3_failed
+ }
+ return True
+
+
+def format_vims(testcase):
+ """
+ Structure:
+ details.sig_test.result.[{result}]
+ details.sig_test.duration
+ details.vIMS.duration
+ details.orchestrator.duration
+
+ Find data for these fields
+ -> details.sig_test.duration
+ -> details.sig_test.tests
+ -> details.sig_test.failures
+ -> details.sig_test.passed
+ -> details.sig_test.skipped
+ -> details.vIMS.duration
+ -> details.orchestrator.duration
+ """
+ testcase_details = testcase['details']
+ test_results = _get_statistics(testcase_details['sig_test']['result'],
+ ('result',),
+ ('Passed', 'Skipped', 'Failed'))
+ passed = test_results['Passed']
+ skipped = test_results['Skipped']
+ failures = test_results['Failed']
+ all_tests = passed + skipped + failures
+ testcase['details'] = {
+ 'sig_test': {
+ 'duration': testcase_details['sig_test']['duration'],
+ 'tests': all_tests,
+ 'failures': failures,
+ 'passed': passed,
+ 'skipped': skipped
+ },
+ 'vIMS': {
+ 'duration': testcase_details['vIMS']['duration']
+ },
+ 'orchestrator': {
+ 'duration': testcase_details['orchestrator']['duration']
+ }
+ }
+ return True
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py
new file mode 100644
index 000000000..25b5320d7
--- /dev/null
+++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py
@@ -0,0 +1,243 @@
+#! /usr/bin/env python
+
+import datetime
+import json
+import os
+import subprocess
+import traceback
+import urlparse
+import uuid
+
+import argparse
+
+from common import logger_utils, elastic_access
+from conf import testcases
+from conf.config import APIConfig
+from mongo2elastic import format
+
+logger = logger_utils.DashboardLogger('mongo2elastic').get
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-c", "--config-file",
+ dest='config_file',
+ help="Config file location")
+parser.add_argument('-ld', '--latest-days',
+ default=0,
+ type=int,
+ metavar='N',
+ help='get entries old at most N days from mongodb and'
+ ' parse those that are not already in elasticsearch.'
+ ' If not present, will get everything from mongodb, which is the default')
+
+args = parser.parse_args()
+CONF = APIConfig().parse(args.config_file)
+
+
+tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4())
+
+
+class DocumentPublisher:
+
+ def __init__(self, doc, fmt, exist_docs, creds, to):
+ self.doc = doc
+ self.fmt = fmt
+ self.creds = creds
+ self.exist_docs = exist_docs
+ self.to = to
+ self.is_formatted = True
+
+ def format(self):
+ try:
+ if self._verify_document() and self.fmt:
+ self.is_formatted = vars(format)[self.fmt](self.doc)
+ else:
+ self.is_formatted = False
+ except Exception:
+ logger.error("Fail in format testcase[%s]\nerror message: %s" %
+ (self.doc, traceback.format_exc()))
+ self.is_formatted = False
+ finally:
+ return self
+
+ def publish(self):
+ if self.is_formatted and self.doc not in self.exist_docs:
+ self._publish()
+
+ def _publish(self):
+ status, data = elastic_access.publish_json(self.doc, self.creds, self.to)
+ if status > 300:
+ logger.error('Publish record[{}] failed, due to [{}]'
+ .format(self.doc, json.loads(data)['error']['reason']))
+
+ def _fix_date(self, date_string):
+ if isinstance(date_string, dict):
+ return date_string['$date']
+ else:
+ return date_string[:-3].replace(' ', 'T') + 'Z'
+
+ def _verify_document(self):
+ """
+ Mandatory fields:
+ installer
+ pod_name
+ version
+ case_name
+ date
+ project
+ details
+
+ these fields must be present and must NOT be None
+
+ Optional fields:
+ description
+
+ these fields will be preserved if the are NOT None
+ """
+ mandatory_fields = ['installer',
+ 'pod_name',
+ 'version',
+ 'case_name',
+ 'project_name',
+ 'details']
+ mandatory_fields_to_modify = {'start_date': self._fix_date}
+ fields_to_swap_or_add = {'scenario': 'version'}
+ if '_id' in self.doc:
+ mongo_id = self.doc['_id']
+ else:
+ mongo_id = None
+ optional_fields = ['description']
+ for key, value in self.doc.items():
+ if key in mandatory_fields:
+ if value is None:
+ # empty mandatory field, invalid input
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
+ " for mandatory field '{}'".format(mongo_id, key))
+ return False
+ else:
+ mandatory_fields.remove(key)
+ elif key in mandatory_fields_to_modify:
+ if value is None:
+ # empty mandatory field, invalid input
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
+ " for mandatory field '{}'".format(mongo_id, key))
+ return False
+ else:
+ self.doc[key] = mandatory_fields_to_modify[key](value)
+ del mandatory_fields_to_modify[key]
+ elif key in fields_to_swap_or_add:
+ if value is None:
+ swapped_key = fields_to_swap_or_add[key]
+ swapped_value = self.doc[swapped_key]
+ logger.info("Swapping field '{}' with value None for '{}' with value '{}'.".format(key, swapped_key,
+ swapped_value))
+ self.doc[key] = swapped_value
+ del fields_to_swap_or_add[key]
+ else:
+ del fields_to_swap_or_add[key]
+ elif key in optional_fields:
+ if value is None:
+ # empty optional field, remove
+ del self.doc[key]
+ optional_fields.remove(key)
+ else:
+ # unknown field
+ del self.doc[key]
+
+ if len(mandatory_fields) > 0:
+ # some mandatory fields are missing
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
+ " mandatory field(s) '{}'".format(mongo_id, mandatory_fields))
+ return False
+ elif len(mandatory_fields_to_modify) > 0:
+ # some mandatory fields are missing
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
+ " mandatory field(s) '{}'".format(mongo_id, mandatory_fields_to_modify.keys()))
+ return False
+ else:
+ if len(fields_to_swap_or_add) > 0:
+ for key, swap_key in fields_to_swap_or_add.iteritems():
+ self.doc[key] = self.doc[swap_key]
+
+ return True
+
+
+class DocumentsPublisher:
+
+ def __init__(self, project, case, fmt, days, elastic_url, creds, to):
+ self.project = project
+ self.case = case
+ self.fmt = fmt
+ self.days = days
+ self.elastic_url = elastic_url
+ self.creds = creds
+ self.to = to
+ self.existed_docs = []
+
+ def export(self):
+ if self.days > 0:
+ past_time = datetime.datetime.today() - datetime.timedelta(days=self.days)
+ query = '''{{
+ "project_name": "{}",
+ "case_name": "{}",
+ "start_date": {{"$gt" : "{}"}}
+ }}'''.format(self.project, self.case, past_time)
+ else:
+ query = '''{{
+ "project_name": "{}",
+ "case_name": "{}"
+ }}'''.format(self.project, self.case)
+ cmd = ['mongoexport',
+ '--db', 'test_results_collection',
+ '--collection', 'results',
+ '--query', '{}'.format(query),
+ '--out', '{}'.format(tmp_docs_file)]
+ try:
+ subprocess.check_call(cmd)
+ return self
+ except Exception, err:
+ logger.error("export mongodb failed: %s" % err)
+ self._remove()
+ exit(-1)
+
+ def get_existed_docs(self):
+ self.existed_docs = elastic_access.get_elastic_docs_by_days(self.elastic_url, self.creds, self.days)
+ return self
+
+ def publish(self):
+ try:
+ with open(tmp_docs_file) as fdocs:
+ for doc_line in fdocs:
+ DocumentPublisher(json.loads(doc_line),
+ self.fmt,
+ self.existed_docs,
+ self.creds,
+ self.to).format().publish()
+ finally:
+ fdocs.close()
+ self._remove()
+
+ def _remove(self):
+ if os.path.exists(tmp_docs_file):
+ os.remove(tmp_docs_file)
+
+
+def main():
+ base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic')
+ to = CONF.destination
+ days = args.latest_days
+ es_creds = CONF.elastic_creds
+
+ if to == 'elasticsearch':
+ to = base_elastic_url
+
+ for project, case_dicts in testcases.testcases_yaml.items():
+ for case_dict in case_dicts:
+ case = case_dict.get('name')
+ fmt = testcases.compose_format(case_dict.get('format'))
+ DocumentsPublisher(project,
+ case,
+ fmt,
+ days,
+ base_elastic_url,
+ es_creds,
+ to).export().get_existed_docs().publish()
diff --git a/utils/test/dashboard/dashboard/mongo2elastic_main.py b/utils/test/dashboard/dashboard/mongo2elastic_main.py
new file mode 100644
index 000000000..141d8f3ab
--- /dev/null
+++ b/utils/test/dashboard/dashboard/mongo2elastic_main.py
@@ -0,0 +1,4 @@
+from mongo2elastic.main import main
+
+if __name__ == '__main__':
+ main()
diff --git a/utils/test/dashboard/etc/config.ini b/utils/test/dashboard/etc/config.ini
new file mode 100644
index 000000000..b94ac7b4f
--- /dev/null
+++ b/utils/test/dashboard/etc/config.ini
@@ -0,0 +1,14 @@
+# to add a new parameter in the config file,
+# the CONF object in config.ini must be updated
+[elastic]
+url = http://localhost:9200
+creds =
+
+[output]
+# elasticsearch or console
+destination = elasticsearch
+
+[kibana]
+url = http://10.63.243.17:5601/app/kibana
+js = true
+js_path = /usr/share/nginx/html/kibana_dashboards/conf.js
diff --git a/utils/test/dashboard/kibana_cleanup.py b/utils/test/dashboard/kibana_cleanup.py
new file mode 100644
index 000000000..9ce4994f5
--- /dev/null
+++ b/utils/test/dashboard/kibana_cleanup.py
@@ -0,0 +1,41 @@
+#! /usr/bin/env python
+import logging
+import urlparse
+
+import argparse
+
+from dashboard.common import elastic_access
+
+logger = logging.getLogger('clear_kibana')
+logger.setLevel(logging.DEBUG)
+file_handler = logging.FileHandler('/var/log/{}.log'.format('clear_kibana'))
+file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
+logger.addHandler(file_handler)
+
+
+def delete_all(url, es_creds):
+ ids = elastic_access.get_elastic_docs(url, es_creds, body=None, field='_id')
+ for id in ids:
+ del_url = '/'.join([url, id])
+ elastic_access.delete_request(del_url, es_creds)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Delete saved kibana searches, visualizations and dashboards')
+ parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
+ help='the url of elasticsearch, defaults to http://localhost:9200')
+
+ parser.add_argument('-u', '--elasticsearch-username', default=None,
+ help='The username with password for elasticsearch in format username:password')
+
+ args = parser.parse_args()
+ base_elastic_url = args.elasticsearch_url
+ es_creds = args.elasticsearch_username
+
+ urls = (urlparse.urljoin(base_elastic_url, '/.kibana/visualization'),
+ urlparse.urljoin(base_elastic_url, '/.kibana/dashboard'),
+ urlparse.urljoin(base_elastic_url, '/.kibana/search'))
+
+ for url in urls:
+ delete_all(url, es_creds)
+
diff --git a/utils/test/scripts/backup-db.sh b/utils/test/scripts/backup-db.sh
deleted file mode 100644
index 35c3fbe5a..000000000
--- a/utils/test/scripts/backup-db.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# SPDX-license-identifier: Apache-2.0
-##############################################################################
-# Copyright (c) 2016 Orange and others.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-echo "Backup Test collection DB"
-now=$(date +"%m_%d_%Y_%H_%M_%S")
-echo $now
-echo " ------------- "
-TARGET_DIR=./$now
-TEST_RESULT_DB_BACKUP="test_collection_db."$now".tar.gz"
-
-echo "Create Directory for backup"
-mkdir -p $TARGET_DIR
-
-echo "Export results"
-mongoexport --db test_results_collection -c results --out $TARGET_DIR/backup-results.json
-echo "Export test cases"
-mongoexport --db test_results_collection -c testcases --out $TARGET_DIR/backup-cases.json
-echo "Export projects"
-mongoexport --db test_results_collection -c projects --out $TARGET_DIR/backup-projects.json
-echo "Export pods"
-mongoexport --db test_results_collection -c pods --out $TARGET_DIR/backup-pod.json
-
-echo "Create tar.gz"
-#tar -cvzf $TEST_RESULT_DB_BACKUP $TARGET_DIR
-
-echo "Delete temp directory"
-#rm -Rf $TARGET_DIR
diff --git a/utils/test/scripts/config.ini b/utils/test/scripts/config.ini
deleted file mode 100644
index 63d283dc8..000000000
--- a/utils/test/scripts/config.ini
+++ /dev/null
@@ -1,14 +0,0 @@
-# to add a new parameter in the config file,
-# the CONF object in config.ini must be updated
-[elastic]
-url = http://localhost:9200
-creds =
-
-[output]
-# elasticsearch or console
-destination = elasticsearch
-
-[kibana]
-url = http://10.63.243.17/kibana/app/kibana
-js = true
-js_path = /usr/share/nginx/html/kibana_dashboards/conf.js
diff --git a/utils/test/scripts/config.py b/utils/test/scripts/config.py
deleted file mode 100644
index 2d447a7ba..000000000
--- a/utils/test/scripts/config.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#! /usr/bin/env python
-
-from ConfigParser import SafeConfigParser, NoOptionError
-
-
-class ParseError(Exception):
- """
- Custom exception class for config file
- """
-
- def __init__(self, message):
- self.msg = message
-
- def __str__(self):
- return 'error parsing config file : %s' % self.msg
-
-
-class APIConfig:
- """
- The purpose of this class is to load values correctly from the config file.
- Each key is declared as an attribute in __init__() and linked in parse()
- """
-
- def __init__(self):
- self._default_config_location = "./config.ini"
- self.elastic_url = 'http://localhost:9200'
- self.elastic_creds = None
- self.destination = 'elasticsearch'
- self.kibana_url = None
- self.is_js = True
- self.js_path = None
-
- def _get_str_parameter(self, section, param):
- try:
- return self._parser.get(section, param)
- except NoOptionError:
- raise ParseError("[%s.%s] parameter not found" % (section, param))
-
- def _get_int_parameter(self, section, param):
- try:
- return int(self._get_str_parameter(section, param))
- except ValueError:
- raise ParseError("[%s.%s] not an int" % (section, param))
-
- def _get_bool_parameter(self, section, param):
- result = self._get_str_parameter(section, param)
- if str(result).lower() == 'true':
- return True
- if str(result).lower() == 'false':
- return False
-
- raise ParseError(
- "[%s.%s : %s] not a boolean" % (section, param, result))
-
- @staticmethod
- def parse(config_location=None):
- obj = APIConfig()
-
- if config_location is None:
- config_location = obj._default_config_location
-
- obj._parser = SafeConfigParser()
- obj._parser.read(config_location)
- if not obj._parser:
- raise ParseError("%s not found" % config_location)
-
- # Linking attributes to keys from file with their sections
- obj.elastic_url = obj._get_str_parameter("elastic", "url")
- obj.elastic_creds = obj._get_str_parameter("elastic", "creds")
- obj.destination = obj._get_str_parameter("output", "destination")
- obj.kibana_url = obj._get_str_parameter("kibana", "url")
- obj.is_js = obj._get_bool_parameter("kibana", "js")
- obj.js_path = obj._get_str_parameter("kibana", "js_path")
-
- return obj
-
- def __str__(self):
- return "elastic_url = %s \n" \
- "elastic_creds = %s \n" \
- "destination = %s \n" \
- "kibana_url = %s \n" \
- "is_js = %s \n" \
- "js_path = %s \n" % (self.elastic_url,
- self.elastic_creds,
- self.destination,
- self.kibana_url,
- self.is_js,
- self.js_path)
diff --git a/utils/test/scripts/create_kibana_dashboards.py b/utils/test/scripts/create_kibana_dashboards.py
deleted file mode 100644
index 19d5b5e52..000000000
--- a/utils/test/scripts/create_kibana_dashboards.py
+++ /dev/null
@@ -1,375 +0,0 @@
-#! /usr/bin/env python
-import json
-import urlparse
-
-import argparse
-
-import logger_utils
-import shared_utils
-import testcases_parser
-from config import APIConfig
-
-logger = logger_utils.KibanaDashboardLogger('elastic2kibana').get
-
-parser = argparse.ArgumentParser()
-parser.add_argument("-c", "--config-file",
- dest='config_file',
- help="Config file location")
-
-args = parser.parse_args()
-CONF = APIConfig().parse(args.config_file)
-
-_installers = {'fuel', 'apex', 'compass', 'joid'}
-
-
-class KibanaDashboard(dict):
- def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
- super(KibanaDashboard, self).__init__()
- self.project_name = project_name
- self.case_name = case_name
- self.family = family
- self.installer = installer
- self.pod = pod
- self.scenarios = scenarios
- self.visualization = visualization
- self._visualization_title = None
- self._kibana_visualizations = []
- self._kibana_dashboard = None
- self._create_visualizations()
- self._create()
-
- def _create_visualizations(self):
- for scenario in self.scenarios:
- self._kibana_visualizations.append(KibanaVisualization(self.project_name,
- self.case_name,
- self.installer,
- self.pod,
- scenario,
- self.visualization))
-
- self._visualization_title = self._kibana_visualizations[0].vis_state_title
-
- def _publish_visualizations(self):
- for visualization in self._kibana_visualizations:
- url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
- logger.debug("publishing visualization '{}'".format(url))
- shared_utils.publish_json(visualization, es_creds, url)
-
- def _construct_panels(self):
- size_x = 6
- size_y = 3
- max_columns = 7
- column = 1
- row = 1
- panel_index = 1
- panels_json = []
- for visualization in self._kibana_visualizations:
- panels_json.append({
- "id": visualization.id,
- "type": 'visualization',
- "panelIndex": panel_index,
- "size_x": size_x,
- "size_y": size_y,
- "col": column,
- "row": row
- })
- panel_index += 1
- column += size_x
- if column > max_columns:
- column = 1
- row += size_y
- return json.dumps(panels_json, separators=(',', ':'))
-
- def _create(self):
- self['title'] = '{} {} {} {} {}'.format(self.project_name,
- self.case_name,
- self.installer,
- self._visualization_title,
- self.pod)
- self.id = self['title'].replace(' ', '-').replace('/', '-')
-
- self['hits'] = 0
- self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
- " pod '{}'".format(self.project_name,
- self.case_name,
- self.installer,
- self._visualization_title,
- self.pod)
- self['panelsJSON'] = self._construct_panels()
- self['optionsJSON'] = json.dumps({
- "darkTheme": False
- },
- separators=(',', ':'))
- self['uiStateJSON'] = "{}"
- self['scenario'] = 1
- self['timeRestore'] = False
- self['kibanaSavedObjectMeta'] = {
- 'searchSourceJSON': json.dumps({
- "filter": [
- {
- "query": {
- "query_string": {
- "query": "*",
- "analyze_wildcard": True
- }
- }
- }
- ]
- },
- separators=(',', ':'))
- }
-
- label = self.case_name
- if 'label' in self.visualization:
- label += " %s" % self.visualization.get('label')
- label += " %s" % self.visualization.get('name')
- self['metadata'] = {
- "label": label,
- "test_family": self.family
- }
-
- def _publish(self):
- url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
- logger.debug("publishing dashboard '{}'".format(url))
- shared_utils.publish_json(self, es_creds, url)
-
- def publish(self):
- self._publish_visualizations()
- self._publish()
-
-
-class KibanaSearchSourceJSON(dict):
- """
- "filter": [
- {"match": {"installer": {"query": installer, "type": "phrase"}}},
- {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
- {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
- ]
- """
-
- def __init__(self, project_name, case_name, installer, pod, scenario):
- super(KibanaSearchSourceJSON, self).__init__()
- self["filter"] = [
- {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
- {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
- {"match": {"installer": {"query": installer, "type": "phrase"}}},
- {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
- ]
- if pod != 'all':
- self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
-
-
-class VisualizationState(dict):
- def __init__(self, visualization):
- super(VisualizationState, self).__init__()
- name = visualization.get('name')
- fields = visualization.get('fields')
-
- if name == 'tests_failures':
- mode = 'grouped'
- metric_type = 'sum'
- self['type'] = 'histogram'
- else:
- # duration or success_percentage
- mode = 'stacked'
- metric_type = 'avg'
- self['type'] = 'line'
-
- self['params'] = {
- "shareYAxis": True,
- "addTooltip": True,
- "addLegend": True,
- "smoothLines": False,
- "scale": "linear",
- "interpolate": "linear",
- "mode": mode,
- "times": [],
- "addTimeMarker": False,
- "defaultYExtents": False,
- "setYExtents": False,
- "yAxis": {}
- }
-
- self['aggs'] = []
-
- i = 1
- for field in fields:
- self['aggs'].append({
- "id": str(i),
- "type": metric_type,
- "schema": "metric",
- "params": {
- "field": field.get('field')
- }
- })
- i += 1
-
- self['aggs'].append({
- "id": str(i),
- "type": 'date_histogram',
- "schema": "segment",
- "params": {
- "field": "start_date",
- "interval": "auto",
- "customInterval": "2h",
- "min_doc_count": 1,
- "extended_bounds": {}
- }
- })
-
- self['listeners'] = {}
- self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
- if x['schema'] == 'metric'])
-
-
-class KibanaVisualization(dict):
- def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
- """
- We need two things
- 1. filter created from
- project_name
- case_name
- installer
- pod
- scenario
- 2. visualization state
- field for y axis (metric) with type (avg, sum, etc.)
- field for x axis (segment) with type (date_histogram)
-
- :return:
- """
- super(KibanaVisualization, self).__init__()
- vis_state = VisualizationState(visualization)
- self.vis_state_title = vis_state['title']
- self['title'] = '{} {} {} {} {} {}'.format(project_name,
- case_name,
- self.vis_state_title,
- installer,
- pod,
- scenario)
- self.id = self['title'].replace(' ', '-').replace('/', '-')
- self['visState'] = json.dumps(vis_state, separators=(',', ':'))
- self['uiStateJSON'] = "{}"
- self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
- " pod '{}' and scenario '{}'".format(project_name,
- case_name,
- self.vis_state_title,
- installer,
- pod,
- scenario)
- self['scenario'] = 1
- self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
- case_name,
- installer,
- pod,
- scenario),
- separators=(',', ':'))}
-
-
-def _get_pods_and_scenarios(project_name, case_name, installer):
- query_json = json.JSONEncoder().encode({
- "query": {
- "bool": {
- "must": [
- {"match_all": {}}
- ],
- "filter": [
- {"match": {"installer": {"query": installer, "type": "phrase"}}},
- {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
- {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
- ]
- }
- }
- })
-
- elastic_data = shared_utils.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
- es_creds, query_json)
-
- pods_and_scenarios = {}
-
- for data in elastic_data:
- pod = data['pod_name']
- if pod in pods_and_scenarios:
- pods_and_scenarios[pod].add(data['scenario'])
- else:
- pods_and_scenarios[pod] = {data['scenario']}
-
- if 'all' in pods_and_scenarios:
- pods_and_scenarios['all'].add(data['scenario'])
- else:
- pods_and_scenarios['all'] = {data['scenario']}
-
- return pods_and_scenarios
-
-
-def construct_dashboards():
- """
- iterate over testcase and installer
- 1. get available pods for each testcase/installer pair
- 2. get available scenario for each testcase/installer/pod tuple
- 3. construct KibanaInput and append
-
- :return: list of KibanaDashboards
- """
- kibana_dashboards = []
- for project, case_dicts in testcases_parser.testcases_yaml.items():
- for case in case_dicts:
- case_name = case.get('name')
- visualizations = case.get('visualizations')
- family = case.get('test_family')
- for installer in _installers:
- pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
- for visualization in visualizations:
- for pod, scenarios in pods_and_scenarios.iteritems():
- kibana_dashboards.append(KibanaDashboard(project,
- case_name,
- family,
- installer,
- pod,
- scenarios,
- visualization))
- return kibana_dashboards
-
-
-def generate_js_inputs(js_file_path, kibana_url, dashboards):
- js_dict = {}
- for dashboard in dashboards:
- dashboard_meta = dashboard['metadata']
- test_family = dashboard_meta['test_family']
- test_label = dashboard_meta['label']
-
- if test_family not in js_dict:
- js_dict[test_family] = {}
-
- js_test_family = js_dict[test_family]
-
- if test_label not in js_test_family:
- js_test_family[test_label] = {}
-
- js_test_label = js_test_family[test_label]
-
- if dashboard.installer not in js_test_label:
- js_test_label[dashboard.installer] = {}
-
- js_installer = js_test_label[dashboard.installer]
- js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
-
- with open(js_file_path, 'w+') as js_file_fdesc:
- js_file_fdesc.write('var kibana_dashboard_links = ')
- js_file_fdesc.write(str(js_dict).replace("u'", "'"))
-
-
-if __name__ == '__main__':
- base_elastic_url = CONF.elastic_url
- generate_inputs = CONF.is_js
- input_file_path = CONF.js_path
- kibana_url = CONF.kibana_url
- es_creds = CONF.elastic_creds
-
- dashboards = construct_dashboards()
-
- for kibana_dashboard in dashboards:
- kibana_dashboard.publish()
-
- if generate_inputs:
- generate_js_inputs(input_file_path, kibana_url, dashboards)
diff --git a/utils/test/scripts/kibana_cleanup.py b/utils/test/scripts/kibana_cleanup.py
deleted file mode 100644
index d87d9a285..000000000
--- a/utils/test/scripts/kibana_cleanup.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#! /usr/bin/env python
-import logging
-import urlparse
-
-import argparse
-
-import shared_utils
-
-logger = logging.getLogger('clear_kibana')
-logger.setLevel(logging.DEBUG)
-file_handler = logging.FileHandler('/var/log/{}.log'.format('clear_kibana'))
-file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
-logger.addHandler(file_handler)
-
-
-def delete_all(url, es_creds):
- ids = shared_utils.get_elastic_docs(url, es_creds, body=None, field='_id')
- for id in ids:
- del_url = '/'.join([url, id])
- shared_utils.delete_request(del_url, es_creds)
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description='Delete saved kibana searches, visualizations and dashboards')
- parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
- help='the url of elasticsearch, defaults to http://localhost:9200')
-
- parser.add_argument('-u', '--elasticsearch-username', default=None,
- help='The username with password for elasticsearch in format username:password')
-
- args = parser.parse_args()
- base_elastic_url = args.elasticsearch_url
- es_creds = args.elasticsearch_username
-
- urls = (urlparse.urljoin(base_elastic_url, '/.kibana/visualization'),
- urlparse.urljoin(base_elastic_url, '/.kibana/dashboard'),
- urlparse.urljoin(base_elastic_url, '/.kibana/search'))
-
- for url in urls:
- delete_all(url, es_creds)
-
diff --git a/utils/test/scripts/logger_utils.py b/utils/test/scripts/logger_utils.py
deleted file mode 100644
index 25d28a582..000000000
--- a/utils/test/scripts/logger_utils.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-#
-# feng.xiaowei@zte.com.cn
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Logging levels:
-# Level Numeric value
-# CRITICAL 50
-# ERROR 40
-# WARNING 30
-# INFO 20
-# DEBUG 10
-# NOTSET 0
-#
-# Usage:
-# import functest_logger as fl
-# logger = fl.Logger("script_name").getLogger()
-# logger.info("message to be shown with - INFO - ")
-# logger.debug("message to be shown with - DEBUG -")
-
-import logging
-import os
-
-
-class Logger(object):
- file_path = '/var/log'
- formatter = logging.Formatter('%(asctime)s - %(name)s - '
- '%(levelname)s - %(message)s')
-
- def __init__(self, logger_name):
-
- IF_DEBUG = os.getenv('IF_DEBUG')
-
- self.logger_name = logger_name
- self.logger = logging.getLogger(logger_name)
- self.logger.propagate = 0
- self.logger.setLevel(logging.DEBUG)
-
- ch = logging.StreamHandler()
- ch.setFormatter(self.formatter)
- if IF_DEBUG is not None and IF_DEBUG.lower() == "true":
- ch.setLevel(logging.DEBUG)
- else:
- ch.setLevel(logging.INFO)
- self.logger.addHandler(ch)
-
- hdlr = logging.FileHandler('%s/%s.log' % (self.file_path, logger_name))
- hdlr.setFormatter(self.formatter)
- hdlr.setLevel(logging.DEBUG)
- self.logger.addHandler(hdlr)
-
- @property
- def get(self):
- return self.logger
-
-
-class KibanaDashboardLogger(Logger):
- file_path = '/var/log/kibana_dashboard'
-
- def __init__(self, logger_name):
- super(KibanaDashboardLogger, self).__init__(logger_name)
-
diff --git a/utils/test/scripts/mongo2elastic_format.py b/utils/test/scripts/mongo2elastic_format.py
deleted file mode 100644
index ef485bae0..000000000
--- a/utils/test/scripts/mongo2elastic_format.py
+++ /dev/null
@@ -1,186 +0,0 @@
-#! /usr/bin/env python
-
-
-def _convert_value(value):
- return value if value != '' else 0
-
-
-def _convert_duration(duration):
- if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration:
- hours, minutes, seconds = duration.split(":")
- hours = _convert_value(hours)
- minutes = _convert_value(minutes)
- seconds = _convert_value(seconds)
- int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds)
- else:
- int_duration = duration
- return int_duration
-
-
-def format_normal(testcase):
- """
- Look for these and leave any of those:
- details.duration
- details.tests
- details.failures
-
- If none are present, then return False
- """
- found = False
- testcase_details = testcase['details']
- fields = ['duration', 'tests', 'failures']
- if isinstance(testcase_details, dict):
- for key, value in testcase_details.items():
- if key in fields:
- found = True
- if key == 'duration':
- testcase_details[key] = _convert_duration(value)
- else:
- del testcase_details[key]
-
- if 'tests' in testcase_details and 'failures' in testcase_details:
- testcase_tests = float(testcase_details['tests'])
- testcase_failures = float(testcase_details['failures'])
- if testcase_tests != 0:
- testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests
- else:
- testcase_details['success_percentage'] = 0
-
-
- return found
-
-
-def format_rally(testcase):
- """
- Structure:
- details.[{summary.duration}]
- details.[{summary.nb success}]
- details.[{summary.nb tests}]
-
- Find data for these fields
- -> details.duration
- -> details.tests
- -> details.success_percentage
- """
- details = testcase['details']
- summary = None
- for item in details:
- if 'summary' in item:
- summary = item['summary']
-
- if not summary:
- return False
-
- testcase['details'] = {
- 'duration': summary['duration'],
- 'tests': summary['nb tests'],
- 'success_percentage': summary['nb success']
- }
- return True
-
-
-def _get_statistics(orig_data, stat_fields, stat_values=None):
- test_results = {}
- for stat_data in orig_data:
- for field in stat_fields:
- stat_value = stat_data[field]
- if stat_value in test_results:
- test_results[stat_value] += 1
- else:
- test_results[stat_value] = 1
-
- if stat_values is not None:
- for stat_value in stat_values:
- if stat_value not in test_results:
- test_results[stat_value] = 0
-
- return test_results
-
-
-def format_onos(testcase):
- """
- Structure:
- details.FUNCvirNet.duration
- details.FUNCvirNet.status.[{Case result}]
- details.FUNCvirNetL3.duration
- details.FUNCvirNetL3.status.[{Case result}]
-
- Find data for these fields
- -> details.FUNCvirNet.duration
- -> details.FUNCvirNet.tests
- -> details.FUNCvirNet.failures
- -> details.FUNCvirNetL3.duration
- -> details.FUNCvirNetL3.tests
- -> details.FUNCvirNetL3.failures
- """
- testcase_details = testcase['details']
-
- if 'FUNCvirNet' not in testcase_details or 'FUNCvirNetL3' not in testcase_details:
- return False
-
- funcvirnet_details = testcase_details['FUNCvirNet']['status']
- funcvirnet_stats = _get_statistics(funcvirnet_details, ('Case result',), ('PASS', 'FAIL'))
- funcvirnet_passed = funcvirnet_stats['PASS']
- funcvirnet_failed = funcvirnet_stats['FAIL']
- funcvirnet_all = funcvirnet_passed + funcvirnet_failed
-
- funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status']
- funcvirnetl3_stats = _get_statistics(funcvirnetl3_details, ('Case result',), ('PASS', 'FAIL'))
- funcvirnetl3_passed = funcvirnetl3_stats['PASS']
- funcvirnetl3_failed = funcvirnetl3_stats['FAIL']
- funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed
-
- testcase_details['FUNCvirNet'] = {
- 'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']),
- 'tests': funcvirnet_all,
- 'failures': funcvirnet_failed
- }
- testcase_details['FUNCvirNetL3'] = {
- 'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']),
- 'tests': funcvirnetl3_all,
- 'failures': funcvirnetl3_failed
- }
- return True
-
-
-def format_vims(testcase):
- """
- Structure:
- details.sig_test.result.[{result}]
- details.sig_test.duration
- details.vIMS.duration
- details.orchestrator.duration
-
- Find data for these fields
- -> details.sig_test.duration
- -> details.sig_test.tests
- -> details.sig_test.failures
- -> details.sig_test.passed
- -> details.sig_test.skipped
- -> details.vIMS.duration
- -> details.orchestrator.duration
- """
- testcase_details = testcase['details']
- test_results = _get_statistics(testcase_details['sig_test']['result'],
- ('result',),
- ('Passed', 'Skipped', 'Failed'))
- passed = test_results['Passed']
- skipped = test_results['Skipped']
- failures = test_results['Failed']
- all_tests = passed + skipped + failures
- testcase['details'] = {
- 'sig_test': {
- 'duration': testcase_details['sig_test']['duration'],
- 'tests': all_tests,
- 'failures': failures,
- 'passed': passed,
- 'skipped': skipped
- },
- 'vIMS': {
- 'duration': testcase_details['vIMS']['duration']
- },
- 'orchestrator': {
- 'duration': testcase_details['orchestrator']['duration']
- }
- }
- return True
diff --git a/utils/test/scripts/mongo_to_elasticsearch.py b/utils/test/scripts/mongo_to_elasticsearch.py
deleted file mode 100644
index 777eda6ad..000000000
--- a/utils/test/scripts/mongo_to_elasticsearch.py
+++ /dev/null
@@ -1,248 +0,0 @@
-#! /usr/bin/env python
-
-import datetime
-import json
-import os
-import subprocess
-import traceback
-import urlparse
-import uuid
-
-import argparse
-
-import logger_utils
-import mongo2elastic_format
-import shared_utils
-import testcases_parser
-from config import APIConfig
-
-logger = logger_utils.KibanaDashboardLogger('mongo2elastic').get
-
-parser = argparse.ArgumentParser()
-parser.add_argument("-c", "--config-file",
- dest='config_file',
- help="Config file location")
-parser.add_argument('-ld', '--latest-days',
- default=0,
- type=int,
- metavar='N',
- help='get entries old at most N days from mongodb and'
- ' parse those that are not already in elasticsearch.'
- ' If not present, will get everything from mongodb, which is the default')
-
-args = parser.parse_args()
-CONF = APIConfig().parse(args.config_file)
-
-
-tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4())
-
-
-class DocumentPublisher:
-
- def __init__(self, doc, fmt, exist_docs, creds, to):
- self.doc = doc
- self.fmt = fmt
- self.creds = creds
- self.exist_docs = exist_docs
- self.to = to
- self.is_formatted = True
-
- def format(self):
- try:
- if self._verify_document() and self.fmt:
- self.is_formatted = vars(mongo2elastic_format)[self.fmt](self.doc)
- else:
- self.is_formatted = False
- except Exception:
- logger.error("Fail in format testcase[%s]\nerror message: %s" %
- (self.doc, traceback.format_exc()))
- self.is_formatted = False
- finally:
- return self
-
- def publish(self):
- if self.is_formatted and self.doc not in self.exist_docs:
- self._publish()
-
- def _publish(self):
- status, data = shared_utils.publish_json(self.doc, self.creds, self.to)
- if status > 300:
- logger.error('Publish record[{}] failed, due to [{}]'
- .format(self.doc, json.loads(data)['error']['reason']))
-
- def _fix_date(self, date_string):
- if isinstance(date_string, dict):
- return date_string['$date']
- else:
- return date_string[:-3].replace(' ', 'T') + 'Z'
-
- def _verify_document(self):
- """
- Mandatory fields:
- installer
- pod_name
- version
- case_name
- date
- project
- details
-
- these fields must be present and must NOT be None
-
- Optional fields:
- description
-
- these fields will be preserved if the are NOT None
- """
- mandatory_fields = ['installer',
- 'pod_name',
- 'version',
- 'case_name',
- 'project_name',
- 'details']
- mandatory_fields_to_modify = {'start_date': self._fix_date}
- fields_to_swap_or_add = {'scenario': 'version'}
- if '_id' in self.doc:
- mongo_id = self.doc['_id']
- else:
- mongo_id = None
- optional_fields = ['description']
- for key, value in self.doc.items():
- if key in mandatory_fields:
- if value is None:
- # empty mandatory field, invalid input
- logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
- " for mandatory field '{}'".format(mongo_id, key))
- return False
- else:
- mandatory_fields.remove(key)
- elif key in mandatory_fields_to_modify:
- if value is None:
- # empty mandatory field, invalid input
- logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
- " for mandatory field '{}'".format(mongo_id, key))
- return False
- else:
- self.doc[key] = mandatory_fields_to_modify[key](value)
- del mandatory_fields_to_modify[key]
- elif key in fields_to_swap_or_add:
- if value is None:
- swapped_key = fields_to_swap_or_add[key]
- swapped_value = self.doc[swapped_key]
- logger.info("Swapping field '{}' with value None for '{}' with value '{}'.".format(key, swapped_key,
- swapped_value))
- self.doc[key] = swapped_value
- del fields_to_swap_or_add[key]
- else:
- del fields_to_swap_or_add[key]
- elif key in optional_fields:
- if value is None:
- # empty optional field, remove
- del self.doc[key]
- optional_fields.remove(key)
- else:
- # unknown field
- del self.doc[key]
-
- if len(mandatory_fields) > 0:
- # some mandatory fields are missing
- logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
- " mandatory field(s) '{}'".format(mongo_id, mandatory_fields))
- return False
- elif len(mandatory_fields_to_modify) > 0:
- # some mandatory fields are missing
- logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
- " mandatory field(s) '{}'".format(mongo_id, mandatory_fields_to_modify.keys()))
- return False
- else:
- if len(fields_to_swap_or_add) > 0:
- for key, swap_key in fields_to_swap_or_add.iteritems():
- self.doc[key] = self.doc[swap_key]
-
- return True
-
-
-class DocumentsPublisher:
-
- def __init__(self, project, case, fmt, days, elastic_url, creds, to):
- self.project = project
- self.case = case
- self.fmt = fmt
- self.days = days
- self.elastic_url = elastic_url
- self.creds = creds
- self.to = to
- self.existed_docs = []
-
- def export(self):
- if self.days > 0:
- past_time = datetime.datetime.today() - datetime.timedelta(days=self.days)
- query = '''{{
- "project_name": "{}",
- "case_name": "{}",
- "start_date": {{"$gt" : "{}"}}
- }}'''.format(self.project, self.case, past_time)
- else:
- query = '''{{
- "project_name": "{}",
- "case_name": "{}"
- }}'''.format(self.project, self.case)
- cmd = ['mongoexport',
- '--db', 'test_results_collection',
- '--collection', 'results',
- '--query', '{}'.format(query),
- '--out', '{}'.format(tmp_docs_file)]
- try:
- subprocess.check_call(cmd)
- return self
- except Exception, err:
- logger.error("export mongodb failed: %s" % err)
- self._remove()
- exit(-1)
-
- def get_existed_docs(self):
- self.existed_docs = shared_utils.get_elastic_docs_by_days(self.elastic_url, self.creds, self.days)
- return self
-
- def publish(self):
- try:
- with open(tmp_docs_file) as fdocs:
- for doc_line in fdocs:
- DocumentPublisher(json.loads(doc_line),
- self.fmt,
- self.existed_docs,
- self.creds,
- self.to).format().publish()
- finally:
- fdocs.close()
- self._remove()
-
- def _remove(self):
- if os.path.exists(tmp_docs_file):
- os.remove(tmp_docs_file)
-
-
-def main():
- base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic')
- to = CONF.destination
- days = args.latest_days
- es_creds = CONF.elastic_creds
-
- if to == 'elasticsearch':
- to = base_elastic_url
-
- for project, case_dicts in testcases_parser.testcases_yaml.items():
- for case_dict in case_dicts:
- case = case_dict.get('name')
- fmt = testcases_parser.compose_format(case_dict.get('format'))
- DocumentsPublisher(project,
- case,
- fmt,
- days,
- base_elastic_url,
- es_creds,
- to).export().get_existed_docs().publish()
-
-
-if __name__ == '__main__':
- main()
\ No newline at end of file
diff --git a/utils/test/scripts/shared_utils.py b/utils/test/scripts/shared_utils.py
deleted file mode 100644
index e90a17fa3..000000000
--- a/utils/test/scripts/shared_utils.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import json
-
-import urllib3
-
-http = urllib3.PoolManager()
-
-
-def delete_request(url, creds, body=None):
- headers = urllib3.make_headers(basic_auth=creds)
- http.request('DELETE', url, headers=headers, body=body)
-
-
-def publish_json(json_ojb, creds, to):
- json_dump = json.dumps(json_ojb)
- if to == 'stdout':
- print json_dump
- return 200, None
- else:
- headers = urllib3.make_headers(basic_auth=creds)
- result = http.request('POST', to, headers=headers, body=json_dump)
- return result.status, result.data
-
-
-def _get_nr_of_hits(elastic_json):
- return elastic_json['hits']['total']
-
-
-def get_elastic_docs(elastic_url, creds, body=None, field = '_source'):
-
- # 1. get the number of results
- headers = urllib3.make_headers(basic_auth=creds)
- elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size=0', headers=headers, body=body).data)
- print elastic_json
- nr_of_hits = _get_nr_of_hits(elastic_json)
-
- # 2. get all results
- elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size={}'.format(nr_of_hits), headers=headers, body=body).data)
-
- elastic_docs = []
- for hit in elastic_json['hits']['hits']:
- elastic_docs.append(hit[field])
- return elastic_docs
-
-
-def get_elastic_docs_by_days(elastic_url, creds, days):
- if days == 0:
- body = '''{
- "query": {
- "match_all": {}
- }
- }'''
- elif days > 0:
- body = '''{{
- "query" : {{
- "range" : {{
- "start_date" : {{
- "gte" : "now-{}d"
- }}
- }}
- }}
- }}'''.format(days)
- else:
- raise Exception('Update days must be non-negative')
- return get_elastic_docs(elastic_url, creds, body)
diff --git a/utils/test/scripts/testcases.yaml b/utils/test/scripts/testcases.yaml
deleted file mode 100644
index 9c33d2e6b..000000000
--- a/utils/test/scripts/testcases.yaml
+++ /dev/null
@@ -1,138 +0,0 @@
-functest:
- -
- name: tempest_smoke_serial
- format: normal
- test_family: VIM
- visualizations:
- -
- name: duration
- fields:
- - field: details.duration
- -
- name: tests_failures
- fields:
- - field: details.tests
- - field: details.failures
- -
- name: success_percentage
- fields:
- - field: details.success_percentage
- -
- name: rally_sanity
- test_family: VIM
- format: rally
- visualizations:
- -
- name: duration
- fields:
- - field: details.duration
- -
- name: tests_failures
- fields:
- - field: details.tests
- -
- name: success_percentage
- fields:
- - field: details.success_percentage
- -
- name: vping_ssh
- format: normal
- test_family: VIM
- visualizations:
- -
- name: duration
- fields:
- - field: details.duration
- -
- name: vping_userdata
- format: normal
- test_family: VIM
- visualizations:
- -
- name: duration
- fields:
- - field: details.duration
- -
- name: odl
- test_family: Controller
- format: odl
- visualizations:
- -
- name: tests_failures
- fields:
- - field: details.tests
- - field: details.failures
- -
- name: success_percentage
- fields:
- - field: details.success_percentage
- -
- name: onos
- format: onos
- test_family: Controller
- visualizations:
- -
- name: duration
- label: FUNCvirNet
- fields:
- - field: details.FUNCvirNet.duration
- -
- name: duration
- label: FUNCvirNetL3
- fields:
- - field: details.FUNCvirNetL3.duration
- -
- name: tests_failures
- label: FUNCvirNet
- fields:
- - field: details.FUNCvirNet.tests
- - field: details.FUNCvirNet.failures
- -
- name: tests_failures
- label: FUNCvirNetL3
- fields:
- - field: details.FUNCvirNetL3.tests
- - field: details.FUNCvirNetL3.failures
- -
- name: vims
- format: vims
- test_family: Features
- visualizations:
- -
- name: duration
- fields:
- - field: details.vIMS.duration
- - field: details.orchestrator.duration
- - field: details.sig_test.duration
- -
- name: tests_failures
- fields:
- - field: details.sig_test.tests
- - field: details.sig_test.failures
- - field: details.sig_test.passed
- - field: details.sig_test.skipped
-promise:
- -
- name: promise
- format: normal
- test_family: Features
- visualizations:
- -
- name: duration
- fields:
- - field: details.duration
- -
- name: tests_failures
- fields:
- - field: details.tests
- - field: details.failures
-doctor:
- -
- name: doctor-notification
- test_family: Features
- format: normal
- visualizations:
- -
- name: duration
- fields:
- - field: details.duration
diff --git a/utils/test/scripts/testcases_parser.py b/utils/test/scripts/testcases_parser.py
deleted file mode 100644
index cf9599858..000000000
--- a/utils/test/scripts/testcases_parser.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import yaml
-
-
-with open('./testcases.yaml') as f:
- testcases_yaml = yaml.safe_load(f)
-f.close()
-
-
-def compose_format(fmt):
- return 'format_' + fmt.strip()
-
-
-def get_format(project, case):
- testcases = testcases_yaml.get(project)
- if isinstance(testcases, list):
- for case_dict in testcases:
- if case_dict['name'] == case:
- return compose_format(case_dict['format'])
- return None
-
-
-if __name__ == '__main__':
- fmt = get_format('functest', 'vping_ssh')
- print fmt
--
cgit