1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
#! /usr/bin/env python
import json
import urlparse
import argparse
from common import elastic_access
from common import logger_utils
from conf import config
from conf import testcases
from dashboard_assembler import DashboardAssembler
from visualization_assembler import VisualizationsAssembler
logger = logger_utils.DashboardLogger('elastic2kibana').get
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config-file",
dest='config_file',
help="Config file location")
args = parser.parse_args()
CONF = config.APIConfig().parse(args.config_file)
_installers = {'fuel', 'apex', 'compass', 'joid'}
def _get_pods_and_scenarios(project_name, case_name, installer):
query_json = json.JSONEncoder().encode({
"query": {
"bool": {
"must": [
{"match_all": {}}
],
"filter": [
{"match": {"installer": {"query": installer, "type": "phrase"}}},
{"match": {"project_name": {"query": project_name, "type": "phrase"}}},
{"match": {"case_name": {"query": case_name, "type": "phrase"}}}
]
}
}
})
elastic_data = elastic_access.get_docs(urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'),
CONF.es_creds,
query_json)
pods_and_scenarios = {}
for data in elastic_data:
pod = data['pod_name']
if pod in pods_and_scenarios:
pods_and_scenarios[pod].add(data['scenario'])
else:
pods_and_scenarios[pod] = {data['scenario']}
if 'all' in pods_and_scenarios:
pods_and_scenarios['all'].add(data['scenario'])
else:
pods_and_scenarios['all'] = {data['scenario']}
return pods_and_scenarios
def construct_dashboards():
"""
iterate over testcase and installer
1. get available pods for each testcase/installer pair
2. get available scenario for each testcase/installer/pod tuple
3. construct KibanaInput and append
:return: list of KibanaDashboards
"""
dashboards = []
for project, case_dicts in testcases.testcases_yaml.items():
for case in case_dicts:
case_name = case.get('name')
vis_ps = case.get('visualizations')
family = case.get('test_family')
for installer in _installers:
pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
for vis_p in vis_ps:
for pod, scenarios in pods_and_scenarios.iteritems():
vissAssember = VisualizationsAssembler(project,
case_name,
installer,
pod,
scenarios,
vis_p,
CONF.es_url,
CONF.es_creds)
dashboardAssembler = DashboardAssembler(project,
case_name,
family,
installer,
pod,
vissAssember.visAssemblers,
CONF.es_url,
CONF.es_creds)
dashboards.append(dashboardAssembler)
return dashboards
def generate_js_inputs(js_file_path, kibana_url, dashboards):
js_dict = {}
for dashboard in dashboards:
dashboard_meta = dashboard.dashboard['metadata']
test_family = dashboard_meta['test_family']
test_label = dashboard_meta['label']
if test_family not in js_dict:
js_dict[test_family] = {}
js_test_family = js_dict[test_family]
if test_label not in js_test_family:
js_test_family[test_label] = {}
js_test_label = js_test_family[test_label]
if dashboard.installer not in js_test_label:
js_test_label[dashboard.installer] = {}
js_installer = js_test_label[dashboard.installer]
js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
with open(js_file_path, 'w+') as js_file_fdesc:
js_file_fdesc.write('var kibana_dashboard_links = ')
js_file_fdesc.write(str(js_dict).replace("u'", "'"))
def main():
dashboards = construct_dashboards()
if CONF.is_js:
generate_js_inputs(CONF.js_path, CONF.kibana_url, dashboards)
|
