summaryrefslogtreecommitdiffstats
path: root/utils
diff options
context:
space:
mode:
authorSerenaFeng <feng.xiaowei@zte.com.cn>2016-10-08 16:45:49 +0800
committerSerenaFeng <feng.xiaowei@zte.com.cn>2016-10-10 10:40:00 +0800
commit8bc61e16a7e61211978539dfb609ed02e0efef25 (patch)
treee0ab419336e40730d7c7794bc576f8c3742e276e /utils
parent26db2b8f2eca5c9856e5fabb6c0f0d25a17c3e8f (diff)
separate visualization building from dashboard building
JIRA: RELENG-155 Change-Id: I19355898d5dc5ef73fcfe4ebaf4604c4bf1e8f10 Signed-off-by: SerenaFeng <feng.xiaowei@zte.com.cn>
Diffstat (limited to 'utils')
-rw-r--r--utils/test/dashboard/dashboard/common/elastic_access.py6
-rw-r--r--utils/test/dashboard/dashboard/conf/config.py12
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py59
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/main.py195
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/utility.py15
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py107
-rw-r--r--utils/test/dashboard/dashboard/mongo2elastic/main.py4
7 files changed, 225 insertions, 173 deletions
diff --git a/utils/test/dashboard/dashboard/common/elastic_access.py b/utils/test/dashboard/dashboard/common/elastic_access.py
index 8c6494d39..aaf776f7a 100644
--- a/utils/test/dashboard/dashboard/common/elastic_access.py
+++ b/utils/test/dashboard/dashboard/common/elastic_access.py
@@ -1,4 +1,5 @@
import json
+import urlparse
import urllib3
@@ -43,3 +44,8 @@ def get_docs(url, creds=None, body=None, field='_source'):
for hit in res_data['hits']['hits']:
docs.append(hit[field])
return docs
+
+
+def publish_kibana(url, creds, type, id, body):
+ url = urlparse.urljoin(url, '/.kibana/{}/{}'.format(type, id))
+ publish_docs(url, creds, body)
diff --git a/utils/test/dashboard/dashboard/conf/config.py b/utils/test/dashboard/dashboard/conf/config.py
index b868999a2..143b1939a 100644
--- a/utils/test/dashboard/dashboard/conf/config.py
+++ b/utils/test/dashboard/dashboard/conf/config.py
@@ -23,8 +23,8 @@ class APIConfig:
def __init__(self):
self._default_config_location = "../etc/config.ini"
- self.elastic_url = 'http://localhost:9200'
- self.elastic_creds = None
+ self.es_url = 'http://localhost:9200'
+ self.es_creds = None
self.kibana_url = None
self.is_js = True
self.js_path = None
@@ -64,8 +64,8 @@ class APIConfig:
raise ParseError("%s not found" % config_location)
# Linking attributes to keys from file with their sections
- obj.elastic_url = obj._get_str_parameter("elastic", "url")
- obj.elastic_creds = obj._get_str_parameter("elastic", "creds")
+ obj.es_url = obj._get_str_parameter("elastic", "url")
+ obj.es_creds = obj._get_str_parameter("elastic", "creds")
obj.kibana_url = obj._get_str_parameter("kibana", "url")
obj.is_js = obj._get_bool_parameter("kibana", "js")
obj.js_path = obj._get_str_parameter("kibana", "js_path")
@@ -77,8 +77,8 @@ class APIConfig:
"elastic_creds = %s \n" \
"kibana_url = %s \n" \
"is_js = %s \n" \
- "js_path = %s \n" % (self.elastic_url,
- self.elastic_creds,
+ "js_path = %s \n" % (self.es_url,
+ self.es_creds,
self.kibana_url,
self.is_js,
self.js_path)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
new file mode 100644
index 000000000..c1e9dfb22
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
@@ -0,0 +1,59 @@
+import json
+
+import utility
+from common import elastic_access
+
+
+class DashboardAssembler(object):
+ def __init__(self,
+ project,
+ case,
+ family,
+ installer,
+ pod,
+ visAssemblers,
+ es_url,
+ es_creds):
+ super(DashboardAssembler, self).__init__()
+ self.project = project
+ self.case = case
+ self.test_family = family
+ self.installer = installer
+ self.pod = pod
+ self.visAssemblers = visAssemblers
+ self.es_url = es_url
+ self.es_creds = es_creds
+ self._assemble()
+ self._publish()
+
+ def _assemble(self):
+ db = {
+ "query": {
+ "project_name": self.project,
+ "case_name": self.case,
+ "installer": self.installer,
+ "metric": self.visAssemblers[0].vis_state_title,
+ "pod": self.pod
+ },
+ "test_family": self.test_family,
+ "ids": [visualization.id for visualization in self.visAssemblers]
+ }
+ template = utility.env.get_template('dashboard.json')
+ self.dashboard = json.loads(template.render(db=db))
+ utility.dumps(self.dashboard,
+ ['description',
+ 'uiStateJSON',
+ 'panelsJSON',
+ 'optionsJSON'])
+ utility.dumps_2depth(self.dashboard,
+ 'kibanaSavedObjectMeta',
+ 'searchSourceJSON')
+ self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
+ return self
+
+ def _publish(self):
+ elastic_access.publish_kibana(self.es_url,
+ self.es_creds,
+ 'dashboard',
+ self.id,
+ self.dashboard)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py
index ae5cbe8fa..8be0a01dd 100644
--- a/utils/test/dashboard/dashboard/elastic2kibana/main.py
+++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py
@@ -3,12 +3,13 @@ import json
import urlparse
import argparse
-from jinja2 import PackageLoader, Environment
from common import elastic_access
from common import logger_utils
+from conf import config
from conf import testcases
-from conf.config import APIConfig
+from dashboard_assembler import DashboardAssembler
+from visualization_assembler import VisualizationsAssembler
logger = logger_utils.DashboardLogger('elastic2kibana').get
@@ -18,154 +19,10 @@ parser.add_argument("-c", "--config-file",
help="Config file location")
args = parser.parse_args()
-CONF = APIConfig().parse(args.config_file)
-base_elastic_url = CONF.elastic_url
-generate_inputs = CONF.is_js
-input_file_path = CONF.js_path
-kibana_url = CONF.kibana_url
-es_creds = CONF.elastic_creds
+CONF = config.APIConfig().parse(args.config_file)
_installers = {'fuel', 'apex', 'compass', 'joid'}
-env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
-env.filters['jsonify'] = json.dumps
-
-
-def dumps(self, items):
- for key in items:
- self.visualization[key] = json.dumps(self.visualization[key])
-
-
-def dumps_2depth(self, key1, key2):
- self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
-
-
-class Dashboard(dict):
- def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
- super(Dashboard, self).__init__()
- self.project_name = project_name
- self.case_name = case_name
- self.family = family
- self.installer = installer
- self.pod = pod
- self.scenarios = scenarios
- self.visualization = visualization
- self._visualization_title = None
- self._kibana_visualizations = []
- self._kibana_dashboard = None
- self._create_visualizations()
- self._create()
-
- def _create_visualizations(self):
- for scenario in self.scenarios:
- self._kibana_visualizations.append(Visualization(self.project_name,
- self.case_name,
- self.installer,
- self.pod,
- scenario,
- self.visualization))
-
- self._visualization_title = self._kibana_visualizations[0].vis_state_title
-
- def _publish_visualizations(self):
- for visualization in self._kibana_visualizations:
- url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
- logger.debug("publishing visualization '{}'".format(url))
- # logger.error("_publish_visualization: %s" % visualization)
- elastic_access.publish_docs(url, es_creds, visualization)
-
- def _create(self):
- db = {
- "query": {
- "project_name": self.project_name,
- "case_name": self.case_name,
- "installer": self.installer,
- "metric": self._visualization_title,
- "pod": self.pod
- },
- "test_family": self.family,
- "ids": [visualization.id for visualization in self._kibana_visualizations]
- }
- template = env.get_template('dashboard.json')
- self.dashboard = json.loads(template.render(db=db))
- dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON'])
- dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON')
- self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
-
-
- def _publish(self):
- url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
- logger.debug("publishing dashboard '{}'".format(url))
- #logger.error("dashboard: %s" % json.dumps(self.dashboard))
- elastic_access.publish_docs(url, es_creds, self.dashboard)
-
- def publish(self):
- self._publish_visualizations()
- self._publish()
-
-
-class VisStateBuilder(object):
- def __init__(self, visualization):
- super(VisStateBuilder, self).__init__()
- self.visualization = visualization
-
- def build(self):
- name = self.visualization.get('name')
- fields = self.visualization.get('fields')
-
- aggs = []
- index = 1
- for field in fields:
- aggs.append({
- "id": index,
- "field": field.get("field")
- })
- index += 1
-
- template = env.get_template('{}.json'.format(name))
- vis = template.render(aggs=aggs)
- return json.loads(vis)
-
-
-class Visualization(object):
- def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
- """
- We need two things
- 1. filter created from
- project_name
- case_name
- installer
- pod
- scenario
- 2. visualization state
- field for y axis (metric) with type (avg, sum, etc.)
- field for x axis (segment) with type (date_histogram)
-
- :return:
- """
- super(Visualization, self).__init__()
- visState = VisStateBuilder(visualization).build()
- self.vis_state_title = visState['title']
-
- vis = {
- "visState": json.dumps(visState),
- "filters": {
- "project_name": project_name,
- "case_name": case_name,
- "installer": installer,
- "metric": self.vis_state_title,
- "pod_name": pod,
- "scenario": scenario
- }
- }
-
- template = env.get_template('visualization.json')
-
- self.visualization = json.loads(template.render(vis=vis))
- dumps(self.visualization, ['visState', 'description', 'uiStateJSON'])
- dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON')
- self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
-
def _get_pods_and_scenarios(project_name, case_name, installer):
query_json = json.JSONEncoder().encode({
@@ -183,8 +40,8 @@ def _get_pods_and_scenarios(project_name, case_name, installer):
}
})
- elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
- es_creds,
+ elastic_data = elastic_access.get_docs(urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'),
+ CONF.es_creds,
query_json)
pods_and_scenarios = {}
@@ -213,24 +70,35 @@ def construct_dashboards():
:return: list of KibanaDashboards
"""
- kibana_dashboards = []
+ dashboards = []
for project, case_dicts in testcases.testcases_yaml.items():
for case in case_dicts:
case_name = case.get('name')
- visualizations = case.get('visualizations')
+ vis_ps = case.get('visualizations')
family = case.get('test_family')
for installer in _installers:
pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
- for visualization in visualizations:
+ for vis_p in vis_ps:
for pod, scenarios in pods_and_scenarios.iteritems():
- kibana_dashboards.append(Dashboard(project,
- case_name,
- family,
- installer,
- pod,
- scenarios,
- visualization))
- return kibana_dashboards
+ vissAssember = VisualizationsAssembler(project,
+ case_name,
+ installer,
+ pod,
+ scenarios,
+ vis_p,
+ CONF.es_url,
+ CONF.es_creds)
+ dashboardAssembler = DashboardAssembler(project,
+ case_name,
+ family,
+ installer,
+ pod,
+ vissAssember.visAssemblers,
+ CONF.es_url,
+ CONF.es_creds)
+ dashboards.append(dashboardAssembler)
+
+ return dashboards
def generate_js_inputs(js_file_path, kibana_url, dashboards):
@@ -264,8 +132,5 @@ def generate_js_inputs(js_file_path, kibana_url, dashboards):
def main():
dashboards = construct_dashboards()
- for kibana_dashboard in dashboards:
- kibana_dashboard.publish()
-
- if generate_inputs:
- generate_js_inputs(input_file_path, kibana_url, dashboards)
+ if CONF.is_js:
+ generate_js_inputs(CONF.js_path, CONF.kibana_url, dashboards)
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/utility.py b/utils/test/dashboard/dashboard/elastic2kibana/utility.py
new file mode 100644
index 000000000..dccd28aed
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/utility.py
@@ -0,0 +1,15 @@
+import json
+
+from jinja2 import Environment, PackageLoader
+
+env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
+env.filters['jsonify'] = json.dumps
+
+
+def dumps(a_dict, items):
+ for key in items:
+ a_dict[key] = json.dumps(a_dict[key])
+
+
+def dumps_2depth(a_dict, key1, key2):
+ a_dict[key1][key2] = json.dumps(a_dict[key1][key2])
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py
new file mode 100644
index 000000000..e3b6b0d38
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/visualization_assembler.py
@@ -0,0 +1,107 @@
+import json
+
+import utility
+from common import elastic_access
+
+
+class VisStateBuilder(object):
+ def __init__(self, vis_p):
+ super(VisStateBuilder, self).__init__()
+ self.vis_p = vis_p
+
+ def build(self):
+ name = self.vis_p.get('name')
+ fields = self.vis_p.get('fields')
+
+ aggs = []
+ index = 1
+ for field in fields:
+ aggs.append({
+ "id": index,
+ "field": field.get("field")
+ })
+ index += 1
+
+ template = utility.env.get_template('{}.json'.format(name))
+ vis = template.render(aggs=aggs)
+ return json.loads(vis)
+
+
+class VisualizationAssembler(object):
+ def __init__(self,
+ project,
+ case,
+ installer,
+ pod,
+ scenario,
+ vis_p,
+ es_url,
+ es_creds):
+ super(VisualizationAssembler, self).__init__()
+ self.project = project
+ self.case = case
+ self.installer = installer
+ self.pod = pod
+ self.scenario = scenario
+ self.vis_p = vis_p
+ self.es_url = es_url
+ self.es_creds = es_creds
+ self._assemble()
+ self._publish()
+
+ def _assemble(self):
+ visState = VisStateBuilder(self.vis_p).build()
+ self.vis_state_title = visState['title']
+
+ vis = {
+ "visState": json.dumps(visState),
+ "filters": {
+ "project_name": self.project,
+ "case_name": self.case,
+ "installer": self.installer,
+ "metric": self.vis_state_title,
+ "pod_name": self.pod,
+ "scenario": self.scenario
+ }
+ }
+
+ template = utility.env.get_template('visualization.json')
+
+ self.visualization = json.loads(template.render(vis=vis))
+ utility.dumps(self.visualization,
+ ['visState', 'description', 'uiStateJSON'])
+ utility.dumps_2depth(self.visualization,
+ 'kibanaSavedObjectMeta',
+ 'searchSourceJSON')
+ title = self.visualization['title']
+ self.id = title.replace(' ', '-').replace('/', '-')
+
+ def _publish(self):
+ elastic_access.publish_kibana(self.es_url,
+ self.es_creds,
+ 'visualization',
+ self.id,
+ self.visualization)
+
+
+class VisualizationsAssembler(object):
+ def __init__(self,
+ project,
+ case,
+ installer,
+ pod,
+ scenarios,
+ vis_p,
+ es_url,
+ es_creds):
+ super(VisualizationsAssembler, self).__init__()
+ self.visAssemblers = []
+ for scenario in scenarios:
+ self.visAssemblers.append(VisualizationAssembler(project,
+ case,
+ installer,
+ pod,
+ scenario,
+ vis_p,
+ es_url,
+ es_creds))
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py
index 76efb14f0..b13f8a7f6 100644
--- a/utils/test/dashboard/dashboard/mongo2elastic/main.py
+++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py
@@ -250,9 +250,9 @@ class DocumentsPublisher:
def main():
- base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic')
+ base_elastic_url = urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic')
days = args.latest_days
- es_creds = CONF.elastic_creds
+ es_creds = CONF.es_creds
for project, case_dicts in testcases.testcases_yaml.items():
for case_dict in case_dicts: