diff options
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/backup-db.sh | 33 | ||||
-rw-r--r-- | scripts/config.ini | 14 | ||||
-rw-r--r-- | scripts/config.py | 88 | ||||
-rw-r--r-- | scripts/create_kibana_dashboards.py | 375 | ||||
-rw-r--r-- | scripts/kibana_cleanup.py | 41 | ||||
-rw-r--r-- | scripts/logger_utils.py | 65 | ||||
-rw-r--r-- | scripts/mongo2elastic_format.py | 186 | ||||
-rw-r--r-- | scripts/mongo_to_elasticsearch.py | 248 | ||||
-rw-r--r-- | scripts/shared_utils.py | 64 | ||||
-rw-r--r-- | scripts/testcases.yaml | 138 | ||||
-rw-r--r-- | scripts/testcases_parser.py | 24 |
11 files changed, 0 insertions, 1276 deletions
diff --git a/scripts/backup-db.sh b/scripts/backup-db.sh deleted file mode 100644 index 35c3fbe..0000000 --- a/scripts/backup-db.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# SPDX-license-identifier: Apache-2.0 -############################################################################## -# Copyright (c) 2016 Orange and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -echo "Backup Test collection DB" -now=$(date +"%m_%d_%Y_%H_%M_%S") -echo $now -echo " ------------- " -TARGET_DIR=./$now -TEST_RESULT_DB_BACKUP="test_collection_db."$now".tar.gz" - -echo "Create Directory for backup" -mkdir -p $TARGET_DIR - -echo "Export results" -mongoexport --db test_results_collection -c results --out $TARGET_DIR/backup-results.json -echo "Export test cases" -mongoexport --db test_results_collection -c testcases --out $TARGET_DIR/backup-cases.json -echo "Export projects" -mongoexport --db test_results_collection -c projects --out $TARGET_DIR/backup-projects.json -echo "Export pods" -mongoexport --db test_results_collection -c pods --out $TARGET_DIR/backup-pod.json - -echo "Create tar.gz" -#tar -cvzf $TEST_RESULT_DB_BACKUP $TARGET_DIR - -echo "Delete temp directory" -#rm -Rf $TARGET_DIR diff --git a/scripts/config.ini b/scripts/config.ini deleted file mode 100644 index 63d283d..0000000 --- a/scripts/config.ini +++ /dev/null @@ -1,14 +0,0 @@ -# to add a new parameter in the config file, -# the CONF object in config.ini must be updated -[elastic] -url = http://localhost:9200 -creds = - -[output] -# elasticsearch or console -destination = elasticsearch - -[kibana] -url = http://10.63.243.17/kibana/app/kibana -js = true -js_path = /usr/share/nginx/html/kibana_dashboards/conf.js diff --git a/scripts/config.py b/scripts/config.py deleted file mode 100644 index 2d447a7..0000000 --- a/scripts/config.py +++ /dev/null @@ -1,88 +0,0 @@ -#! /usr/bin/env python - -from ConfigParser import SafeConfigParser, NoOptionError - - -class ParseError(Exception): - """ - Custom exception class for config file - """ - - def __init__(self, message): - self.msg = message - - def __str__(self): - return 'error parsing config file : %s' % self.msg - - -class APIConfig: - """ - The purpose of this class is to load values correctly from the config file. - Each key is declared as an attribute in __init__() and linked in parse() - """ - - def __init__(self): - self._default_config_location = "./config.ini" - self.elastic_url = 'http://localhost:9200' - self.elastic_creds = None - self.destination = 'elasticsearch' - self.kibana_url = None - self.is_js = True - self.js_path = None - - def _get_str_parameter(self, section, param): - try: - return self._parser.get(section, param) - except NoOptionError: - raise ParseError("[%s.%s] parameter not found" % (section, param)) - - def _get_int_parameter(self, section, param): - try: - return int(self._get_str_parameter(section, param)) - except ValueError: - raise ParseError("[%s.%s] not an int" % (section, param)) - - def _get_bool_parameter(self, section, param): - result = self._get_str_parameter(section, param) - if str(result).lower() == 'true': - return True - if str(result).lower() == 'false': - return False - - raise ParseError( - "[%s.%s : %s] not a boolean" % (section, param, result)) - - @staticmethod - def parse(config_location=None): - obj = APIConfig() - - if config_location is None: - config_location = obj._default_config_location - - obj._parser = SafeConfigParser() - obj._parser.read(config_location) - if not obj._parser: - raise ParseError("%s not found" % config_location) - - # Linking attributes to keys from file with their sections - obj.elastic_url = obj._get_str_parameter("elastic", "url") - obj.elastic_creds = obj._get_str_parameter("elastic", "creds") - obj.destination = obj._get_str_parameter("output", "destination") - obj.kibana_url = obj._get_str_parameter("kibana", "url") - obj.is_js = obj._get_bool_parameter("kibana", "js") - obj.js_path = obj._get_str_parameter("kibana", "js_path") - - return obj - - def __str__(self): - return "elastic_url = %s \n" \ - "elastic_creds = %s \n" \ - "destination = %s \n" \ - "kibana_url = %s \n" \ - "is_js = %s \n" \ - "js_path = %s \n" % (self.elastic_url, - self.elastic_creds, - self.destination, - self.kibana_url, - self.is_js, - self.js_path) diff --git a/scripts/create_kibana_dashboards.py b/scripts/create_kibana_dashboards.py deleted file mode 100644 index 19d5b5e..0000000 --- a/scripts/create_kibana_dashboards.py +++ /dev/null @@ -1,375 +0,0 @@ -#! /usr/bin/env python -import json -import urlparse - -import argparse - -import logger_utils -import shared_utils -import testcases_parser -from config import APIConfig - -logger = logger_utils.KibanaDashboardLogger('elastic2kibana').get - -parser = argparse.ArgumentParser() -parser.add_argument("-c", "--config-file", - dest='config_file', - help="Config file location") - -args = parser.parse_args() -CONF = APIConfig().parse(args.config_file) - -_installers = {'fuel', 'apex', 'compass', 'joid'} - - -class KibanaDashboard(dict): - def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization): - super(KibanaDashboard, self).__init__() - self.project_name = project_name - self.case_name = case_name - self.family = family - self.installer = installer - self.pod = pod - self.scenarios = scenarios - self.visualization = visualization - self._visualization_title = None - self._kibana_visualizations = [] - self._kibana_dashboard = None - self._create_visualizations() - self._create() - - def _create_visualizations(self): - for scenario in self.scenarios: - self._kibana_visualizations.append(KibanaVisualization(self.project_name, - self.case_name, - self.installer, - self.pod, - scenario, - self.visualization)) - - self._visualization_title = self._kibana_visualizations[0].vis_state_title - - def _publish_visualizations(self): - for visualization in self._kibana_visualizations: - url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) - logger.debug("publishing visualization '{}'".format(url)) - shared_utils.publish_json(visualization, es_creds, url) - - def _construct_panels(self): - size_x = 6 - size_y = 3 - max_columns = 7 - column = 1 - row = 1 - panel_index = 1 - panels_json = [] - for visualization in self._kibana_visualizations: - panels_json.append({ - "id": visualization.id, - "type": 'visualization', - "panelIndex": panel_index, - "size_x": size_x, - "size_y": size_y, - "col": column, - "row": row - }) - panel_index += 1 - column += size_x - if column > max_columns: - column = 1 - row += size_y - return json.dumps(panels_json, separators=(',', ':')) - - def _create(self): - self['title'] = '{} {} {} {} {}'.format(self.project_name, - self.case_name, - self.installer, - self._visualization_title, - self.pod) - self.id = self['title'].replace(' ', '-').replace('/', '-') - - self['hits'] = 0 - self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \ - " pod '{}'".format(self.project_name, - self.case_name, - self.installer, - self._visualization_title, - self.pod) - self['panelsJSON'] = self._construct_panels() - self['optionsJSON'] = json.dumps({ - "darkTheme": False - }, - separators=(',', ':')) - self['uiStateJSON'] = "{}" - self['scenario'] = 1 - self['timeRestore'] = False - self['kibanaSavedObjectMeta'] = { - 'searchSourceJSON': json.dumps({ - "filter": [ - { - "query": { - "query_string": { - "query": "*", - "analyze_wildcard": True - } - } - } - ] - }, - separators=(',', ':')) - } - - label = self.case_name - if 'label' in self.visualization: - label += " %s" % self.visualization.get('label') - label += " %s" % self.visualization.get('name') - self['metadata'] = { - "label": label, - "test_family": self.family - } - - def _publish(self): - url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) - logger.debug("publishing dashboard '{}'".format(url)) - shared_utils.publish_json(self, es_creds, url) - - def publish(self): - self._publish_visualizations() - self._publish() - - -class KibanaSearchSourceJSON(dict): - """ - "filter": [ - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}} - ] - """ - - def __init__(self, project_name, case_name, installer, pod, scenario): - super(KibanaSearchSourceJSON, self).__init__() - self["filter"] = [ - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}}, - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"scenario": {"query": scenario, "type": "phrase"}}} - ] - if pod != 'all': - self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}}) - - -class VisualizationState(dict): - def __init__(self, visualization): - super(VisualizationState, self).__init__() - name = visualization.get('name') - fields = visualization.get('fields') - - if name == 'tests_failures': - mode = 'grouped' - metric_type = 'sum' - self['type'] = 'histogram' - else: - # duration or success_percentage - mode = 'stacked' - metric_type = 'avg' - self['type'] = 'line' - - self['params'] = { - "shareYAxis": True, - "addTooltip": True, - "addLegend": True, - "smoothLines": False, - "scale": "linear", - "interpolate": "linear", - "mode": mode, - "times": [], - "addTimeMarker": False, - "defaultYExtents": False, - "setYExtents": False, - "yAxis": {} - } - - self['aggs'] = [] - - i = 1 - for field in fields: - self['aggs'].append({ - "id": str(i), - "type": metric_type, - "schema": "metric", - "params": { - "field": field.get('field') - } - }) - i += 1 - - self['aggs'].append({ - "id": str(i), - "type": 'date_histogram', - "schema": "segment", - "params": { - "field": "start_date", - "interval": "auto", - "customInterval": "2h", - "min_doc_count": 1, - "extended_bounds": {} - } - }) - - self['listeners'] = {} - self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs'] - if x['schema'] == 'metric']) - - -class KibanaVisualization(dict): - def __init__(self, project_name, case_name, installer, pod, scenario, visualization): - """ - We need two things - 1. filter created from - project_name - case_name - installer - pod - scenario - 2. visualization state - field for y axis (metric) with type (avg, sum, etc.) - field for x axis (segment) with type (date_histogram) - - :return: - """ - super(KibanaVisualization, self).__init__() - vis_state = VisualizationState(visualization) - self.vis_state_title = vis_state['title'] - self['title'] = '{} {} {} {} {} {}'.format(project_name, - case_name, - self.vis_state_title, - installer, - pod, - scenario) - self.id = self['title'].replace(' ', '-').replace('/', '-') - self['visState'] = json.dumps(vis_state, separators=(',', ':')) - self['uiStateJSON'] = "{}" - self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \ - " pod '{}' and scenario '{}'".format(project_name, - case_name, - self.vis_state_title, - installer, - pod, - scenario) - self['scenario'] = 1 - self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name, - case_name, - installer, - pod, - scenario), - separators=(',', ':'))} - - -def _get_pods_and_scenarios(project_name, case_name, installer): - query_json = json.JSONEncoder().encode({ - "query": { - "bool": { - "must": [ - {"match_all": {}} - ], - "filter": [ - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}} - ] - } - } - }) - - elastic_data = shared_utils.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), - es_creds, query_json) - - pods_and_scenarios = {} - - for data in elastic_data: - pod = data['pod_name'] - if pod in pods_and_scenarios: - pods_and_scenarios[pod].add(data['scenario']) - else: - pods_and_scenarios[pod] = {data['scenario']} - - if 'all' in pods_and_scenarios: - pods_and_scenarios['all'].add(data['scenario']) - else: - pods_and_scenarios['all'] = {data['scenario']} - - return pods_and_scenarios - - -def construct_dashboards(): - """ - iterate over testcase and installer - 1. get available pods for each testcase/installer pair - 2. get available scenario for each testcase/installer/pod tuple - 3. construct KibanaInput and append - - :return: list of KibanaDashboards - """ - kibana_dashboards = [] - for project, case_dicts in testcases_parser.testcases_yaml.items(): - for case in case_dicts: - case_name = case.get('name') - visualizations = case.get('visualizations') - family = case.get('test_family') - for installer in _installers: - pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) - for visualization in visualizations: - for pod, scenarios in pods_and_scenarios.iteritems(): - kibana_dashboards.append(KibanaDashboard(project, - case_name, - family, - installer, - pod, - scenarios, - visualization)) - return kibana_dashboards - - -def generate_js_inputs(js_file_path, kibana_url, dashboards): - js_dict = {} - for dashboard in dashboards: - dashboard_meta = dashboard['metadata'] - test_family = dashboard_meta['test_family'] - test_label = dashboard_meta['label'] - - if test_family not in js_dict: - js_dict[test_family] = {} - - js_test_family = js_dict[test_family] - - if test_label not in js_test_family: - js_test_family[test_label] = {} - - js_test_label = js_test_family[test_label] - - if dashboard.installer not in js_test_label: - js_test_label[dashboard.installer] = {} - - js_installer = js_test_label[dashboard.installer] - js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id - - with open(js_file_path, 'w+') as js_file_fdesc: - js_file_fdesc.write('var kibana_dashboard_links = ') - js_file_fdesc.write(str(js_dict).replace("u'", "'")) - - -if __name__ == '__main__': - base_elastic_url = CONF.elastic_url - generate_inputs = CONF.is_js - input_file_path = CONF.js_path - kibana_url = CONF.kibana_url - es_creds = CONF.elastic_creds - - dashboards = construct_dashboards() - - for kibana_dashboard in dashboards: - kibana_dashboard.publish() - - if generate_inputs: - generate_js_inputs(input_file_path, kibana_url, dashboards) diff --git a/scripts/kibana_cleanup.py b/scripts/kibana_cleanup.py deleted file mode 100644 index d87d9a2..0000000 --- a/scripts/kibana_cleanup.py +++ /dev/null @@ -1,41 +0,0 @@ -#! /usr/bin/env python -import logging -import urlparse - -import argparse - -import shared_utils - -logger = logging.getLogger('clear_kibana') -logger.setLevel(logging.DEBUG) -file_handler = logging.FileHandler('/var/log/{}.log'.format('clear_kibana')) -file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) -logger.addHandler(file_handler) - - -def delete_all(url, es_creds): - ids = shared_utils.get_elastic_docs(url, es_creds, body=None, field='_id') - for id in ids: - del_url = '/'.join([url, id]) - shared_utils.delete_request(del_url, es_creds) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Delete saved kibana searches, visualizations and dashboards') - parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200', - help='the url of elasticsearch, defaults to http://localhost:9200') - - parser.add_argument('-u', '--elasticsearch-username', default=None, - help='The username with password for elasticsearch in format username:password') - - args = parser.parse_args() - base_elastic_url = args.elasticsearch_url - es_creds = args.elasticsearch_username - - urls = (urlparse.urljoin(base_elastic_url, '/.kibana/visualization'), - urlparse.urljoin(base_elastic_url, '/.kibana/dashboard'), - urlparse.urljoin(base_elastic_url, '/.kibana/search')) - - for url in urls: - delete_all(url, es_creds) - diff --git a/scripts/logger_utils.py b/scripts/logger_utils.py deleted file mode 100644 index 25d28a5..0000000 --- a/scripts/logger_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Logging levels: -# Level Numeric value -# CRITICAL 50 -# ERROR 40 -# WARNING 30 -# INFO 20 -# DEBUG 10 -# NOTSET 0 -# -# Usage: -# import functest_logger as fl -# logger = fl.Logger("script_name").getLogger() -# logger.info("message to be shown with - INFO - ") -# logger.debug("message to be shown with - DEBUG -") - -import logging -import os - - -class Logger(object): - file_path = '/var/log' - formatter = logging.Formatter('%(asctime)s - %(name)s - ' - '%(levelname)s - %(message)s') - - def __init__(self, logger_name): - - IF_DEBUG = os.getenv('IF_DEBUG') - - self.logger_name = logger_name - self.logger = logging.getLogger(logger_name) - self.logger.propagate = 0 - self.logger.setLevel(logging.DEBUG) - - ch = logging.StreamHandler() - ch.setFormatter(self.formatter) - if IF_DEBUG is not None and IF_DEBUG.lower() == "true": - ch.setLevel(logging.DEBUG) - else: - ch.setLevel(logging.INFO) - self.logger.addHandler(ch) - - hdlr = logging.FileHandler('%s/%s.log' % (self.file_path, logger_name)) - hdlr.setFormatter(self.formatter) - hdlr.setLevel(logging.DEBUG) - self.logger.addHandler(hdlr) - - @property - def get(self): - return self.logger - - -class KibanaDashboardLogger(Logger): - file_path = '/var/log/kibana_dashboard' - - def __init__(self, logger_name): - super(KibanaDashboardLogger, self).__init__(logger_name) - diff --git a/scripts/mongo2elastic_format.py b/scripts/mongo2elastic_format.py deleted file mode 100644 index ef485ba..0000000 --- a/scripts/mongo2elastic_format.py +++ /dev/null @@ -1,186 +0,0 @@ -#! /usr/bin/env python - - -def _convert_value(value): - return value if value != '' else 0 - - -def _convert_duration(duration): - if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration: - hours, minutes, seconds = duration.split(":") - hours = _convert_value(hours) - minutes = _convert_value(minutes) - seconds = _convert_value(seconds) - int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds) - else: - int_duration = duration - return int_duration - - -def format_normal(testcase): - """ - Look for these and leave any of those: - details.duration - details.tests - details.failures - - If none are present, then return False - """ - found = False - testcase_details = testcase['details'] - fields = ['duration', 'tests', 'failures'] - if isinstance(testcase_details, dict): - for key, value in testcase_details.items(): - if key in fields: - found = True - if key == 'duration': - testcase_details[key] = _convert_duration(value) - else: - del testcase_details[key] - - if 'tests' in testcase_details and 'failures' in testcase_details: - testcase_tests = float(testcase_details['tests']) - testcase_failures = float(testcase_details['failures']) - if testcase_tests != 0: - testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests - else: - testcase_details['success_percentage'] = 0 - - - return found - - -def format_rally(testcase): - """ - Structure: - details.[{summary.duration}] - details.[{summary.nb success}] - details.[{summary.nb tests}] - - Find data for these fields - -> details.duration - -> details.tests - -> details.success_percentage - """ - details = testcase['details'] - summary = None - for item in details: - if 'summary' in item: - summary = item['summary'] - - if not summary: - return False - - testcase['details'] = { - 'duration': summary['duration'], - 'tests': summary['nb tests'], - 'success_percentage': summary['nb success'] - } - return True - - -def _get_statistics(orig_data, stat_fields, stat_values=None): - test_results = {} - for stat_data in orig_data: - for field in stat_fields: - stat_value = stat_data[field] - if stat_value in test_results: - test_results[stat_value] += 1 - else: - test_results[stat_value] = 1 - - if stat_values is not None: - for stat_value in stat_values: - if stat_value not in test_results: - test_results[stat_value] = 0 - - return test_results - - -def format_onos(testcase): - """ - Structure: - details.FUNCvirNet.duration - details.FUNCvirNet.status.[{Case result}] - details.FUNCvirNetL3.duration - details.FUNCvirNetL3.status.[{Case result}] - - Find data for these fields - -> details.FUNCvirNet.duration - -> details.FUNCvirNet.tests - -> details.FUNCvirNet.failures - -> details.FUNCvirNetL3.duration - -> details.FUNCvirNetL3.tests - -> details.FUNCvirNetL3.failures - """ - testcase_details = testcase['details'] - - if 'FUNCvirNet' not in testcase_details or 'FUNCvirNetL3' not in testcase_details: - return False - - funcvirnet_details = testcase_details['FUNCvirNet']['status'] - funcvirnet_stats = _get_statistics(funcvirnet_details, ('Case result',), ('PASS', 'FAIL')) - funcvirnet_passed = funcvirnet_stats['PASS'] - funcvirnet_failed = funcvirnet_stats['FAIL'] - funcvirnet_all = funcvirnet_passed + funcvirnet_failed - - funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status'] - funcvirnetl3_stats = _get_statistics(funcvirnetl3_details, ('Case result',), ('PASS', 'FAIL')) - funcvirnetl3_passed = funcvirnetl3_stats['PASS'] - funcvirnetl3_failed = funcvirnetl3_stats['FAIL'] - funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed - - testcase_details['FUNCvirNet'] = { - 'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']), - 'tests': funcvirnet_all, - 'failures': funcvirnet_failed - } - testcase_details['FUNCvirNetL3'] = { - 'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']), - 'tests': funcvirnetl3_all, - 'failures': funcvirnetl3_failed - } - return True - - -def format_vims(testcase): - """ - Structure: - details.sig_test.result.[{result}] - details.sig_test.duration - details.vIMS.duration - details.orchestrator.duration - - Find data for these fields - -> details.sig_test.duration - -> details.sig_test.tests - -> details.sig_test.failures - -> details.sig_test.passed - -> details.sig_test.skipped - -> details.vIMS.duration - -> details.orchestrator.duration - """ - testcase_details = testcase['details'] - test_results = _get_statistics(testcase_details['sig_test']['result'], - ('result',), - ('Passed', 'Skipped', 'Failed')) - passed = test_results['Passed'] - skipped = test_results['Skipped'] - failures = test_results['Failed'] - all_tests = passed + skipped + failures - testcase['details'] = { - 'sig_test': { - 'duration': testcase_details['sig_test']['duration'], - 'tests': all_tests, - 'failures': failures, - 'passed': passed, - 'skipped': skipped - }, - 'vIMS': { - 'duration': testcase_details['vIMS']['duration'] - }, - 'orchestrator': { - 'duration': testcase_details['orchestrator']['duration'] - } - } - return True diff --git a/scripts/mongo_to_elasticsearch.py b/scripts/mongo_to_elasticsearch.py deleted file mode 100644 index 777eda6..0000000 --- a/scripts/mongo_to_elasticsearch.py +++ /dev/null @@ -1,248 +0,0 @@ -#! /usr/bin/env python - -import datetime -import json -import os -import subprocess -import traceback -import urlparse -import uuid - -import argparse - -import logger_utils -import mongo2elastic_format -import shared_utils -import testcases_parser -from config import APIConfig - -logger = logger_utils.KibanaDashboardLogger('mongo2elastic').get - -parser = argparse.ArgumentParser() -parser.add_argument("-c", "--config-file", - dest='config_file', - help="Config file location") -parser.add_argument('-ld', '--latest-days', - default=0, - type=int, - metavar='N', - help='get entries old at most N days from mongodb and' - ' parse those that are not already in elasticsearch.' - ' If not present, will get everything from mongodb, which is the default') - -args = parser.parse_args() -CONF = APIConfig().parse(args.config_file) - - -tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4()) - - -class DocumentPublisher: - - def __init__(self, doc, fmt, exist_docs, creds, to): - self.doc = doc - self.fmt = fmt - self.creds = creds - self.exist_docs = exist_docs - self.to = to - self.is_formatted = True - - def format(self): - try: - if self._verify_document() and self.fmt: - self.is_formatted = vars(mongo2elastic_format)[self.fmt](self.doc) - else: - self.is_formatted = False - except Exception: - logger.error("Fail in format testcase[%s]\nerror message: %s" % - (self.doc, traceback.format_exc())) - self.is_formatted = False - finally: - return self - - def publish(self): - if self.is_formatted and self.doc not in self.exist_docs: - self._publish() - - def _publish(self): - status, data = shared_utils.publish_json(self.doc, self.creds, self.to) - if status > 300: - logger.error('Publish record[{}] failed, due to [{}]' - .format(self.doc, json.loads(data)['error']['reason'])) - - def _fix_date(self, date_string): - if isinstance(date_string, dict): - return date_string['$date'] - else: - return date_string[:-3].replace(' ', 'T') + 'Z' - - def _verify_document(self): - """ - Mandatory fields: - installer - pod_name - version - case_name - date - project - details - - these fields must be present and must NOT be None - - Optional fields: - description - - these fields will be preserved if the are NOT None - """ - mandatory_fields = ['installer', - 'pod_name', - 'version', - 'case_name', - 'project_name', - 'details'] - mandatory_fields_to_modify = {'start_date': self._fix_date} - fields_to_swap_or_add = {'scenario': 'version'} - if '_id' in self.doc: - mongo_id = self.doc['_id'] - else: - mongo_id = None - optional_fields = ['description'] - for key, value in self.doc.items(): - if key in mandatory_fields: - if value is None: - # empty mandatory field, invalid input - logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value" - " for mandatory field '{}'".format(mongo_id, key)) - return False - else: - mandatory_fields.remove(key) - elif key in mandatory_fields_to_modify: - if value is None: - # empty mandatory field, invalid input - logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value" - " for mandatory field '{}'".format(mongo_id, key)) - return False - else: - self.doc[key] = mandatory_fields_to_modify[key](value) - del mandatory_fields_to_modify[key] - elif key in fields_to_swap_or_add: - if value is None: - swapped_key = fields_to_swap_or_add[key] - swapped_value = self.doc[swapped_key] - logger.info("Swapping field '{}' with value None for '{}' with value '{}'.".format(key, swapped_key, - swapped_value)) - self.doc[key] = swapped_value - del fields_to_swap_or_add[key] - else: - del fields_to_swap_or_add[key] - elif key in optional_fields: - if value is None: - # empty optional field, remove - del self.doc[key] - optional_fields.remove(key) - else: - # unknown field - del self.doc[key] - - if len(mandatory_fields) > 0: - # some mandatory fields are missing - logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing" - " mandatory field(s) '{}'".format(mongo_id, mandatory_fields)) - return False - elif len(mandatory_fields_to_modify) > 0: - # some mandatory fields are missing - logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing" - " mandatory field(s) '{}'".format(mongo_id, mandatory_fields_to_modify.keys())) - return False - else: - if len(fields_to_swap_or_add) > 0: - for key, swap_key in fields_to_swap_or_add.iteritems(): - self.doc[key] = self.doc[swap_key] - - return True - - -class DocumentsPublisher: - - def __init__(self, project, case, fmt, days, elastic_url, creds, to): - self.project = project - self.case = case - self.fmt = fmt - self.days = days - self.elastic_url = elastic_url - self.creds = creds - self.to = to - self.existed_docs = [] - - def export(self): - if self.days > 0: - past_time = datetime.datetime.today() - datetime.timedelta(days=self.days) - query = '''{{ - "project_name": "{}", - "case_name": "{}", - "start_date": {{"$gt" : "{}"}} - }}'''.format(self.project, self.case, past_time) - else: - query = '''{{ - "project_name": "{}", - "case_name": "{}" - }}'''.format(self.project, self.case) - cmd = ['mongoexport', - '--db', 'test_results_collection', - '--collection', 'results', - '--query', '{}'.format(query), - '--out', '{}'.format(tmp_docs_file)] - try: - subprocess.check_call(cmd) - return self - except Exception, err: - logger.error("export mongodb failed: %s" % err) - self._remove() - exit(-1) - - def get_existed_docs(self): - self.existed_docs = shared_utils.get_elastic_docs_by_days(self.elastic_url, self.creds, self.days) - return self - - def publish(self): - try: - with open(tmp_docs_file) as fdocs: - for doc_line in fdocs: - DocumentPublisher(json.loads(doc_line), - self.fmt, - self.existed_docs, - self.creds, - self.to).format().publish() - finally: - fdocs.close() - self._remove() - - def _remove(self): - if os.path.exists(tmp_docs_file): - os.remove(tmp_docs_file) - - -def main(): - base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic') - to = CONF.destination - days = args.latest_days - es_creds = CONF.elastic_creds - - if to == 'elasticsearch': - to = base_elastic_url - - for project, case_dicts in testcases_parser.testcases_yaml.items(): - for case_dict in case_dicts: - case = case_dict.get('name') - fmt = testcases_parser.compose_format(case_dict.get('format')) - DocumentsPublisher(project, - case, - fmt, - days, - base_elastic_url, - es_creds, - to).export().get_existed_docs().publish() - - -if __name__ == '__main__': - main()
\ No newline at end of file diff --git a/scripts/shared_utils.py b/scripts/shared_utils.py deleted file mode 100644 index e90a17f..0000000 --- a/scripts/shared_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -import json - -import urllib3 - -http = urllib3.PoolManager() - - -def delete_request(url, creds, body=None): - headers = urllib3.make_headers(basic_auth=creds) - http.request('DELETE', url, headers=headers, body=body) - - -def publish_json(json_ojb, creds, to): - json_dump = json.dumps(json_ojb) - if to == 'stdout': - print json_dump - return 200, None - else: - headers = urllib3.make_headers(basic_auth=creds) - result = http.request('POST', to, headers=headers, body=json_dump) - return result.status, result.data - - -def _get_nr_of_hits(elastic_json): - return elastic_json['hits']['total'] - - -def get_elastic_docs(elastic_url, creds, body=None, field = '_source'): - - # 1. get the number of results - headers = urllib3.make_headers(basic_auth=creds) - elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size=0', headers=headers, body=body).data) - print elastic_json - nr_of_hits = _get_nr_of_hits(elastic_json) - - # 2. get all results - elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size={}'.format(nr_of_hits), headers=headers, body=body).data) - - elastic_docs = [] - for hit in elastic_json['hits']['hits']: - elastic_docs.append(hit[field]) - return elastic_docs - - -def get_elastic_docs_by_days(elastic_url, creds, days): - if days == 0: - body = '''{ - "query": { - "match_all": {} - } - }''' - elif days > 0: - body = '''{{ - "query" : {{ - "range" : {{ - "start_date" : {{ - "gte" : "now-{}d" - }} - }} - }} - }}'''.format(days) - else: - raise Exception('Update days must be non-negative') - return get_elastic_docs(elastic_url, creds, body) diff --git a/scripts/testcases.yaml b/scripts/testcases.yaml deleted file mode 100644 index 9c33d2e..0000000 --- a/scripts/testcases.yaml +++ /dev/null @@ -1,138 +0,0 @@ -functest: - - - name: tempest_smoke_serial - format: normal - test_family: VIM - visualizations: - - - name: duration - fields: - - field: details.duration - - - name: tests_failures - fields: - - field: details.tests - - field: details.failures - - - name: success_percentage - fields: - - field: details.success_percentage - - - name: rally_sanity - test_family: VIM - format: rally - visualizations: - - - name: duration - fields: - - field: details.duration - - - name: tests_failures - fields: - - field: details.tests - - - name: success_percentage - fields: - - field: details.success_percentage - - - name: vping_ssh - format: normal - test_family: VIM - visualizations: - - - name: duration - fields: - - field: details.duration - - - name: vping_userdata - format: normal - test_family: VIM - visualizations: - - - name: duration - fields: - - field: details.duration - - - name: odl - test_family: Controller - format: odl - visualizations: - - - name: tests_failures - fields: - - field: details.tests - - field: details.failures - - - name: success_percentage - fields: - - field: details.success_percentage - - - name: onos - format: onos - test_family: Controller - visualizations: - - - name: duration - label: FUNCvirNet - fields: - - field: details.FUNCvirNet.duration - - - name: duration - label: FUNCvirNetL3 - fields: - - field: details.FUNCvirNetL3.duration - - - name: tests_failures - label: FUNCvirNet - fields: - - field: details.FUNCvirNet.tests - - field: details.FUNCvirNet.failures - - - name: tests_failures - label: FUNCvirNetL3 - fields: - - field: details.FUNCvirNetL3.tests - - field: details.FUNCvirNetL3.failures - - - name: vims - format: vims - test_family: Features - visualizations: - - - name: duration - fields: - - field: details.vIMS.duration - - field: details.orchestrator.duration - - field: details.sig_test.duration - - - name: tests_failures - fields: - - field: details.sig_test.tests - - field: details.sig_test.failures - - field: details.sig_test.passed - - field: details.sig_test.skipped -promise: - - - name: promise - format: normal - test_family: Features - visualizations: - - - name: duration - fields: - - field: details.duration - - - name: tests_failures - fields: - - field: details.tests - - field: details.failures -doctor: - - - name: doctor-notification - test_family: Features - format: normal - visualizations: - - - name: duration - fields: - - field: details.duration diff --git a/scripts/testcases_parser.py b/scripts/testcases_parser.py deleted file mode 100644 index cf95998..0000000 --- a/scripts/testcases_parser.py +++ /dev/null @@ -1,24 +0,0 @@ -import yaml - - -with open('./testcases.yaml') as f: - testcases_yaml = yaml.safe_load(f) -f.close() - - -def compose_format(fmt): - return 'format_' + fmt.strip() - - -def get_format(project, case): - testcases = testcases_yaml.get(project) - if isinstance(testcases, list): - for case_dict in testcases: - if case_dict['name'] == case: - return compose_format(case_dict['format']) - return None - - -if __name__ == '__main__': - fmt = get_format('functest', 'vping_ssh') - print fmt |