diff options
Diffstat (limited to 'utils/test')
9 files changed, 242 insertions, 146 deletions
diff --git a/utils/test/dashboard/dashboard/common/elastic_access.py b/utils/test/dashboard/dashboard/common/elastic_access.py index e90a17fa3..8c6494d39 100644 --- a/utils/test/dashboard/dashboard/common/elastic_access.py +++ b/utils/test/dashboard/dashboard/common/elastic_access.py @@ -5,60 +5,41 @@ import urllib3 http = urllib3.PoolManager() -def delete_request(url, creds, body=None): +def _request(method, url, creds=None, body=None): headers = urllib3.make_headers(basic_auth=creds) - http.request('DELETE', url, headers=headers, body=body) + return http.request(method, url, headers=headers, body=body) -def publish_json(json_ojb, creds, to): - json_dump = json.dumps(json_ojb) - if to == 'stdout': - print json_dump - return 200, None - else: - headers = urllib3.make_headers(basic_auth=creds) - result = http.request('POST', to, headers=headers, body=json_dump) - return result.status, result.data +def _post(url, creds=None, body=None): + return _request('POST', url, creds=creds, body=body) -def _get_nr_of_hits(elastic_json): - return elastic_json['hits']['total'] +def _get(url, creds=None, body=None): + return json.loads(_request('GET', url, creds=creds, body=body).data) -def get_elastic_docs(elastic_url, creds, body=None, field = '_source'): +def delete_docs(url, creds=None, body=None): + return _request('DELETE', url, creds=creds, body=body) - # 1. get the number of results - headers = urllib3.make_headers(basic_auth=creds) - elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size=0', headers=headers, body=body).data) - print elastic_json - nr_of_hits = _get_nr_of_hits(elastic_json) - - # 2. get all results - elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size={}'.format(nr_of_hits), headers=headers, body=body).data) - - elastic_docs = [] - for hit in elastic_json['hits']['hits']: - elastic_docs.append(hit[field]) - return elastic_docs - - -def get_elastic_docs_by_days(elastic_url, creds, days): - if days == 0: - body = '''{ - "query": { - "match_all": {} - } - }''' - elif days > 0: - body = '''{{ - "query" : {{ - "range" : {{ - "start_date" : {{ - "gte" : "now-{}d" - }} - }} - }} - }}'''.format(days) - else: - raise Exception('Update days must be non-negative') - return get_elastic_docs(elastic_url, creds, body) + +def publish_docs(url, creds=None, body=None): + result = _post(url, creds=creds, body=(json.dumps(body))) + return result.status, result.data + + +def _get_docs_nr(url, creds=None, body=None): + res_data = _get('{}/_search?size=0'.format(url), creds=creds, body=body) + print type(res_data), res_data + return res_data['hits']['total'] + + +def get_docs(url, creds=None, body=None, field='_source'): + + docs_nr = _get_docs_nr(url, creds=creds, body=body) + res_data = _get('{}/_search?size={}'.format(url, docs_nr), + creds=creds, body=body) + + docs = [] + for hit in res_data['hits']['hits']: + docs.append(hit[field]) + return docs diff --git a/utils/test/dashboard/dashboard/conf/config.py b/utils/test/dashboard/dashboard/conf/config.py index 2e0f1cabb..b868999a2 100644 --- a/utils/test/dashboard/dashboard/conf/config.py +++ b/utils/test/dashboard/dashboard/conf/config.py @@ -25,7 +25,6 @@ class APIConfig: self._default_config_location = "../etc/config.ini" self.elastic_url = 'http://localhost:9200' self.elastic_creds = None - self.destination = 'elasticsearch' self.kibana_url = None self.is_js = True self.js_path = None @@ -67,7 +66,6 @@ class APIConfig: # Linking attributes to keys from file with their sections obj.elastic_url = obj._get_str_parameter("elastic", "url") obj.elastic_creds = obj._get_str_parameter("elastic", "creds") - obj.destination = obj._get_str_parameter("output", "destination") obj.kibana_url = obj._get_str_parameter("kibana", "url") obj.is_js = obj._get_bool_parameter("kibana", "js") obj.js_path = obj._get_str_parameter("kibana", "js_path") @@ -77,12 +75,10 @@ class APIConfig: def __str__(self): return "elastic_url = %s \n" \ "elastic_creds = %s \n" \ - "destination = %s \n" \ "kibana_url = %s \n" \ "is_js = %s \n" \ "js_path = %s \n" % (self.elastic_url, - self.elastic_creds, - self.destination, - self.kibana_url, - self.is_js, - self.js_path) + self.elastic_creds, + self.kibana_url, + self.is_js, + self.js_path) diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py index c1cbc308e..95f758ea8 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/main.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py @@ -3,8 +3,10 @@ import json import urlparse import argparse +from jinja2 import PackageLoader, Environment -from common import logger_utils, elastic_access +from common import elastic_access +from common import logger_utils from conf import testcases from conf.config import APIConfig @@ -51,13 +53,14 @@ class KibanaDashboard(dict): scenario, self.visualization)) - self._visualization_title = self._kibana_visualizations[0].vis_state_title + self._visualization_title = self._kibana_visualizations[0].vis_title def _publish_visualizations(self): for visualization in self._kibana_visualizations: url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) logger.debug("publishing visualization '{}'".format(url)) - elastic_access.publish_json(visualization, es_creds, url) + # logger.error("_publish_visualization: %s" % visualization) + elastic_access.publish_docs(url, es_creds, visualization) def _construct_panels(self): size_x = 6 @@ -135,7 +138,7 @@ class KibanaDashboard(dict): def _publish(self): url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) logger.debug("publishing dashboard '{}'".format(url)) - elastic_access.publish_json(self, es_creds, url) + elastic_access.publish_docs(url, es_creds, self) def publish(self): self._publish_visualizations() @@ -163,67 +166,29 @@ class KibanaSearchSourceJSON(dict): self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}}) -class VisualizationState(dict): +class VisualizationBuilder(object): def __init__(self, visualization): - super(VisualizationState, self).__init__() - name = visualization.get('name') - fields = visualization.get('fields') - - if name == 'tests_failures': - mode = 'grouped' - metric_type = 'sum' - self['type'] = 'histogram' - else: - # duration or success_percentage - mode = 'stacked' - metric_type = 'avg' - self['type'] = 'line' - - self['params'] = { - "shareYAxis": True, - "addTooltip": True, - "addLegend": True, - "smoothLines": False, - "scale": "linear", - "interpolate": "linear", - "mode": mode, - "times": [], - "addTimeMarker": False, - "defaultYExtents": False, - "setYExtents": False, - "yAxis": {} - } + super(VisualizationBuilder, self).__init__() + self.visualization = visualization - self['aggs'] = [] + def build(self): + name = self.visualization.get('name') + fields = self.visualization.get('fields') - i = 1 + aggs = [] + index = 1 for field in fields: - self['aggs'].append({ - "id": str(i), - "type": metric_type, - "schema": "metric", - "params": { - "field": field.get('field') - } - }) - i += 1 - - self['aggs'].append({ - "id": str(i), - "type": 'date_histogram', - "schema": "segment", - "params": { - "field": "start_date", - "interval": "auto", - "customInterval": "2h", - "min_doc_count": 1, - "extended_bounds": {} - } + aggs.append({ + "id": index, + "field": field.get("field") }) + index += 1 - self['listeners'] = {} - self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs'] - if x['schema'] == 'metric']) + env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) + env.filters['jsonify'] = json.dumps + template = env.get_template('{}.json'.format(name)) + vis = template.render(aggs=aggs) + return json.loads(vis) class KibanaVisualization(dict): @@ -243,24 +208,24 @@ class KibanaVisualization(dict): :return: """ super(KibanaVisualization, self).__init__() - vis_state = VisualizationState(visualization) - self.vis_state_title = vis_state['title'] + vis = VisualizationBuilder(visualization).build() + self.vis_title = vis['title'] self['title'] = '{} {} {} {} {} {}'.format(project_name, case_name, - self.vis_state_title, + self.vis_title, installer, pod, scenario) self.id = self['title'].replace(' ', '-').replace('/', '-') - self['visState'] = json.dumps(vis_state, separators=(',', ':')) + self['visState'] = json.dumps(vis, separators=(',', ':')) self['uiStateJSON'] = "{}" - self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \ + self['description'] = "Kibana visualization for project_name '{}', case_name '{}', metric '{}', installer '{}'," \ " pod '{}' and scenario '{}'".format(project_name, - case_name, - self.vis_state_title, - installer, - pod, - scenario) + case_name, + self.vis_title, + installer, + pod, + scenario) self['scenario'] = 1 self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name, case_name, @@ -286,7 +251,7 @@ def _get_pods_and_scenarios(project_name, case_name, installer): } }) - elastic_data = elastic_access.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), + elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), es_creds, query_json) pods_and_scenarios = {} diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/duration.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/duration.json new file mode 100644 index 000000000..f50a668db --- /dev/null +++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/duration.json @@ -0,0 +1,45 @@ +{% set aggs = aggs|default([]) -%} + +{ + "title": "duration", + "type": "line", + "listeners": {}, + "params": { + "addLegend": true, + "shareYAxis": true, + "addTooltip": true, + "smoothLines": false, + "scale": "linear", + "interpolate": "linear", + "times": [], + "addTimeMarker": false, + "defaultYExtents": false, + "setYExtents": false, + "yAxis": {}, + "mode": "stacked" + }, + "aggs": [ + {% for agg in aggs %} + { + "id": {{agg.id }}, + "type": "avg", + "schema": "metric", + "params": { + "field": "{{agg.field}}" + } + }, + {% endfor %} + { + "id": {{ aggs|length + 1 }}, + "type": "date_histogram", + "schema": "segment", + "params": { + "field": "start_date", + "interval": "auto", + "customInterval": "2h", + "min_doc_count": 1, + "extended_bounds": {} + } + } + ] +} diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/success_percentage.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/success_percentage.json new file mode 100644 index 000000000..993070844 --- /dev/null +++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/success_percentage.json @@ -0,0 +1,45 @@ +{% set aggs = aggs|default([]) -%} + +{ + "title": "success_percentage", + "type": "line", + "listeners": {}, + "params": { + "addLegend": true, + "shareYAxis": true, + "addTooltip": true, + "smoothLines": false, + "scale": "linear", + "interpolate": "linear", + "times": [], + "addTimeMarker": false, + "defaultYExtents": false, + "setYExtents": false, + "yAxis": {}, + "mode": "stacked" + }, + "aggs": [ + {% for agg in aggs %} + { + "id": {{agg.id }}, + "type": "avg", + "schema": "metric", + "params": { + "field": "{{agg.field}}" + } + }, + {% endfor %} + { + "id": {{ aggs|length + 1 }}, + "type": "date_histogram", + "schema": "segment", + "params": { + "field": "start_date", + "interval": "auto", + "customInterval": "2h", + "min_doc_count": 1, + "extended_bounds": {} + } + } + ] +} diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/tests_failures.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/tests_failures.json new file mode 100644 index 000000000..01f9ba89e --- /dev/null +++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/tests_failures.json @@ -0,0 +1,45 @@ +{% set aggs = aggs|default([]) -%} + +{ + "title": "tests_failures", + "type": "histogram", + "listeners": {}, + "params": { + "addLegend": true, + "shareYAxis": true, + "addTooltip": true, + "smoothLines": false, + "scale": "linear", + "interpolate": "linear", + "times": [], + "addTimeMarker": false, + "defaultYExtents": false, + "setYExtents": false, + "yAxis": {}, + "mode": "grouped" + }, + "aggs": [ + {% for agg in aggs %} + { + "id": {{agg.id }}, + "type": "sum", + "schema": "metric", + "params": { + "field": "{{agg.field}}" + } + }, + {% endfor %} + { + "id": {{ aggs|length + 1 }}, + "type": "date_histogram", + "schema": "segment", + "params": { + "field": "start_date", + "interval": "auto", + "customInterval": "2h", + "min_doc_count": 1, + "extended_bounds": {} + } + } + ] +} diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py index 25b5320d7..76efb14f0 100644 --- a/utils/test/dashboard/dashboard/mongo2elastic/main.py +++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py @@ -38,12 +38,12 @@ tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4()) class DocumentPublisher: - def __init__(self, doc, fmt, exist_docs, creds, to): + def __init__(self, doc, fmt, exist_docs, creds, elastic_url): self.doc = doc self.fmt = fmt self.creds = creds self.exist_docs = exist_docs - self.to = to + self.elastic_url = elastic_url self.is_formatted = True def format(self): @@ -64,7 +64,7 @@ class DocumentPublisher: self._publish() def _publish(self): - status, data = elastic_access.publish_json(self.doc, self.creds, self.to) + status, data = elastic_access.publish_docs(self.elastic_url, self.creds, self.doc) if status > 300: logger.error('Publish record[{}] failed, due to [{}]' .format(self.doc, json.loads(data)['error']['reason'])) @@ -163,14 +163,13 @@ class DocumentPublisher: class DocumentsPublisher: - def __init__(self, project, case, fmt, days, elastic_url, creds, to): + def __init__(self, project, case, fmt, days, elastic_url, creds): self.project = project self.case = case self.fmt = fmt self.days = days self.elastic_url = elastic_url self.creds = creds - self.to = to self.existed_docs = [] def export(self): @@ -200,7 +199,36 @@ class DocumentsPublisher: exit(-1) def get_existed_docs(self): - self.existed_docs = elastic_access.get_elastic_docs_by_days(self.elastic_url, self.creds, self.days) + if self.days == 0: + body = '''{{ + "query": {{ + "bool": {{ + "must": [ + {{ "match": {{ "project_name": "{}" }} }}, + {{ "match": {{ "case_name": "{}" }} }} + ] + }} + }} + }}'''.format(self.project, self.case) + elif self.days > 0: + body = '''{{ + "query": {{ + "bool": {{ + "must": [ + {{ "match": {{ "project_name": "{}" }} }}, + {{ "match": {{ "case_name": "{}" }} }} + ], + "filter": {{ + "range": {{ + "start_date": {{ "gte": "now-{}d" }} + }} + }} + }} + }} + }}'''.format(self.project, self.case, self.days) + else: + raise Exception('Update days must be non-negative') + self.existed_docs = elastic_access.get_docs(self.elastic_url, self.creds, body) return self def publish(self): @@ -211,7 +239,7 @@ class DocumentsPublisher: self.fmt, self.existed_docs, self.creds, - self.to).format().publish() + self.elastic_url).format().publish() finally: fdocs.close() self._remove() @@ -223,13 +251,9 @@ class DocumentsPublisher: def main(): base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic') - to = CONF.destination days = args.latest_days es_creds = CONF.elastic_creds - if to == 'elasticsearch': - to = base_elastic_url - for project, case_dicts in testcases.testcases_yaml.items(): for case_dict in case_dicts: case = case_dict.get('name') @@ -239,5 +263,4 @@ def main(): fmt, days, base_elastic_url, - es_creds, - to).export().get_existed_docs().publish() + es_creds).export().get_existed_docs().publish() diff --git a/utils/test/dashboard/etc/config.ini b/utils/test/dashboard/etc/config.ini index b94ac7b4f..1e67bd822 100644 --- a/utils/test/dashboard/etc/config.ini +++ b/utils/test/dashboard/etc/config.ini @@ -4,10 +4,6 @@ url = http://localhost:9200 creds = -[output] -# elasticsearch or console -destination = elasticsearch - [kibana] url = http://10.63.243.17:5601/app/kibana js = true diff --git a/utils/test/dashboard/kibana_cleanup.py b/utils/test/dashboard/kibana_cleanup.py index 9ce4994f5..ee0190049 100644 --- a/utils/test/dashboard/kibana_cleanup.py +++ b/utils/test/dashboard/kibana_cleanup.py @@ -14,10 +14,10 @@ logger.addHandler(file_handler) def delete_all(url, es_creds): - ids = elastic_access.get_elastic_docs(url, es_creds, body=None, field='_id') + ids = elastic_access.get_docs(url, es_creds, body=None, field='_id') for id in ids: del_url = '/'.join([url, id]) - elastic_access.delete_request(del_url, es_creds) + elastic_access.delete_docs(del_url, es_creds) if __name__ == '__main__': |