From abbd8ce62ba662cd2659589a9996e827d13ad061 Mon Sep 17 00:00:00 2001 From: yuyang Date: Tue, 20 Dec 2016 00:37:59 +0800 Subject: Add flake8 style config and for POSCA JIRA: BOTTLENECK-100 Adding flake8 configuration and fix for POSCA Change-Id: Ib822d5abde535263a4473d67c15cc17995f1417b Signed-off-by: yuyang --- flake8_cfg | 4 + tests.sh | 6 +- .../posca_system_bandwidth_config.json | 4 + .../posca_system_bandwidth_dashboard.json | 12 ++ .../posca_system_bandwidth_discover.json | 5 + .../posca_system_bandwidth_index_pattern.json | 5 + .../posca_system_bandwidth_line_char.json | 10 ++ .../posca_system_bandwidth_line_data.json | 10 ++ .../posca_system_bandwidth_terms_data.json | 10 ++ .../posca/testcase_dashboard/system_bandwidth.py | 152 ++++++++++----------- testsuites/posca/testcase_script/common_script.py | 83 +++++------ .../testcase_script/posca_factor_cpu_burden.py | 24 ++-- .../testcase_script/posca_factor_rx_cache_size.py | 2 +- .../testcase_script/posca_factor_rx_pkt_size.py | 2 +- .../posca_factor_system_bandwidth.py | 25 ++-- .../testcase_script/posca_factor_tx_cache_size.py | 4 +- .../testcase_script/posca_factor_tx_pkt_size.py | 4 +- 17 files changed, 203 insertions(+), 159 deletions(-) create mode 100644 flake8_cfg create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_config.json create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_dashboard.json create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_discover.json create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_index_pattern.json create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_char.json create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_data.json create mode 100644 testsuites/posca/testcase_dashboard/posca_system_bandwidth_terms_data.json diff --git a/flake8_cfg b/flake8_cfg new file mode 100644 index 00000000..578c0556 --- /dev/null +++ b/flake8_cfg @@ -0,0 +1,4 @@ +[flake8] +format = pylint +statistics = True +count = True diff --git a/tests.sh b/tests.sh index 8df8140d..18110770 100755 --- a/tests.sh +++ b/tests.sh @@ -17,9 +17,9 @@ run_flake8() { echo "Running flake8 ... " logfile=test_results.log if [ $FILE_OPTION == "f" ]; then - flake8 ../bottlenecks > $logfile + sudo flake8 --append-config=flake8_cfg testsuites/posca/ > $logfile else - flake8 ../bottlenecks + sudo flake8 --append-config=flake8_cfg testsuites/posca/ fi if [ $? -ne 0 ]; then @@ -40,8 +40,6 @@ run_tests() { run_coverage() { echo "Running coverage ..." - #source tests/ci/cover.sh - #run_coverage_test #will add future } diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_config.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_config.json new file mode 100644 index 00000000..c49c8953 --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_config.json @@ -0,0 +1,4 @@ +{ + "buildNum": 10146, + "defaultIndex": "bottlenecks" +} diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_dashboard.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_dashboard.json new file mode 100644 index 00000000..3ef86b72 --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_dashboard.json @@ -0,0 +1,12 @@ +{ + "title": "system_bandwidth_dashboard", + "hits": 0, + "description": "", + "panelsJSON": "[{\"id\":\"system_bandwidth_line-char\",\"type\":\"visualization\",\"panelIndex\":3,\"size_x\":7,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"system_bandwidth_line-date\",\"type\":\"visualization\",\"panelIndex\":4,\"size_x\":7,\"size_y\":4,\"col\":1,\"row\":5},{\"id\":\"system_bandwidth_terms_data\",\"type\":\"visualization\",\"panelIndex\":5,\"size_x\":5,\"size_y\":8,\"col\":8,\"row\":1}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}" + } +} diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_discover.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_discover.json new file mode 100644 index 00000000..8ca1218f --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_discover.json @@ -0,0 +1,5 @@ +{ + "title": "system_bandwidth", + "description": "", + "version": 1 +} diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_index_pattern.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_index_pattern.json new file mode 100644 index 00000000..53ae1a7a --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_index_pattern.json @@ -0,0 +1,5 @@ +{ + "title": "bottlenecks", + "timeFieldName": "time", + "fields": "[{\"name\":\"runner_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"tx_cache_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"task_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"scenarios\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"local_cpu_util\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rx_cache_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"local_transport_retrans\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"throughput\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"throughput_units\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"mean_latency\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tx_msg_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"deploy_scenario\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"pod_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"target\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"rx_msg_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"installer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"remote_cpu_util\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]" +} diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_char.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_char.json new file mode 100644 index 00000000..13acf16b --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_char.json @@ -0,0 +1,10 @@ +{ + "title": "system_bandwidth_line-char", + "visState": "{\"title\":\"system_bandwidth_line-char\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"throughput\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"rx_msg_size\",\"size\":30,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"tx_msg_size\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_data.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_data.json new file mode 100644 index 00000000..8a98ce9b --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_line_data.json @@ -0,0 +1,10 @@ +{ + "title": "system_bandwidth_line-date", + "visState": "{\"title\":\"system_bandwidth_line-date\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"throughput\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"time\",\"size\":100,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"3\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"rx_msg_size\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/testsuites/posca/testcase_dashboard/posca_system_bandwidth_terms_data.json b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_terms_data.json new file mode 100644 index 00000000..5e8e787b --- /dev/null +++ b/testsuites/posca/testcase_dashboard/posca_system_bandwidth_terms_data.json @@ -0,0 +1,10 @@ +{ + "title": "system_bandwidth_terms_data", + "visState": "{\"title\":\"system_bandwidth_terms_data\",\"type\":\"table\",\"params\":{\"perPage\":80,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"throughput\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"tx_msg_size\",\"size\":200,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"5\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"rx_msg_size\",\"size\":200,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"6\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"mean_latency\",\"size\":5,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"7\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"local_cpu_util\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"_term\"}},{\"id\":\"8\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"remote_cpu_util\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/testsuites/posca/testcase_dashboard/system_bandwidth.py b/testsuites/posca/testcase_dashboard/system_bandwidth.py index 92dc0b5c..63671273 100755 --- a/testsuites/posca/testcase_dashboard/system_bandwidth.py +++ b/testsuites/posca/testcase_dashboard/system_bandwidth.py @@ -10,9 +10,15 @@ import ConfigParser from elasticsearch import Elasticsearch from pyroute2 import IPDB +import json config = ConfigParser.ConfigParser() -file_str = "/home/opnfv/bottlenecks/testsuites/posca/testcase_cfg/posca_factor_system_bandwidth.yaml" + +dashboard_dir = "/home/opnfv/bottlenecks/testsuites/posca\ +/testcase_dashboard/" +file_str = "/home/opnfv/bottlenecks/testsuites/posca/\ +testcase_cfg/posca_factor_system_bandwidth.yaml" + with open(file_str, "rd") as cfgfile: config.readfp(cfgfile) ES_ip_a = config.get("config", "ES_ip") @@ -20,114 +26,98 @@ with open(file_str, "rd") as cfgfile: with IPDB() as ip: GATEWAY_IP = ip.routes['default'].gateway if ES_ip_a is "": - ES_ip_a = GATEWAY_IP+":9200" - print("ES_ip is null get local ip is %s" %(ES_ip_a)) + ES_ip_a = "{}:9200".format(GATEWAY_IP) + print("ES_ip is null get local ip is {}".format(ES_ip_a)) es_ip = ES_ip_a.split(':') -es = Elasticsearch([{'host':es_ip[0]}]) -doc={ - "title": "bottlenecks", - "timeFieldName": "time", - "fields": "[{\"name\":\"runner_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"tx_cache_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"task_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"scenarios\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"local_cpu_util\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rx_cache_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"local_transport_retrans\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"throughput\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"throughput_units\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"mean_latency\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tx_msg_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"deploy_scenario\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"pod_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"target\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"rx_msg_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"installer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"remote_cpu_util\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]" -} +es = Elasticsearch([{'host': es_ip[0]}]) -res = es.index(index=".kibana",doc_type="index-pattern",id="bottlenecks",body=doc) -if res['created']=="True": +# Create bottlenecks index +with open(dashboard_dir + 'posca_system_bandwidth\ +_index_pattern.json') as index_pattern: + doc = json.load(index_pattern) +res = es.index( + index=".kibana", + doc_type="index-pattern", + id="bottlenecks", + body=doc) +if res['created'] == "True": print("bottlenecks index-pattern has created") else: print("bottlenecks index-pattern has existed") -doc={ - "buildNum": 10146, - "defaultIndex": "bottlenecks" -} -res = es.index(index=".kibana",doc_type="config",id="4.6.1",body=doc) - -if res['created']=="True": - print("bottlenecks config has created") +with open(dashboard_dir + 'posca_system_bandwidth\ +_config.json') as index_config: + doc = json.load(index_config) +res = es.index(index=".kibana", doc_type="config", id="4.6.1", body=doc) +if res['created'] == "True": + print("bottlenecks config has created") else: - print("bottlenecks config has existed") + print("bottlenecks config has existed") -doc={ -"title": "system_bandwidth", - "description": "", - "version": 1, -} -res = es.index(index=".kibana",doc_type="search",id="system_bandwidth",body=doc) -if res['created']=="True": +# Configure discover panel +with open(dashboard_dir + 'posca_system_bandwidth\ +_discover.json') as index_discover: + doc = json.load(index_discover) +res = es.index( + index=".kibana", + doc_type="search", + id="system_bandwidth", + body=doc) +if res['created'] == "True": print("system_bandwidth search has created") else: print("system_bandwidth search has existed") -doc = { - "title": "system_bandwidth_line-date", - "visState": "{\"title\":\"system_bandwidth_line-date\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"throughput\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"time\",\"size\":100,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"3\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"rx_msg_size\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } -} +# Create testing data in line graph +with open(dashboard_dir + 'posca_system_bandwidth\ +_line_data.json') as line_data: + doc = json.load(line_data) res = es.index( - index=".kibana", doc_type="visualization", id="system_bandwidth_line-date", body=doc) -if res['created']=="True": + index=".kibana", + doc_type="visualization", + id="system_bandwidth_line-date", + body=doc) +if res['created'] == "True": print("system_bandwidth_line-date visualization has created") else: print("system_bandwidth_line-date visualization has existed") -doc = { - "title": "system_bandwidth_line-char", - "visState": "{\"title\":\"system_bandwidth_line-char\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"throughput\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"rx_msg_size\",\"size\":30,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"tx_msg_size\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } -} - +# Create comparison results in line chart +with open(dashboard_dir + 'posca_system_bandwidth\ +_line_char.json') as line_char: + doc = json.load(line_char) res = es.index( - index=".kibana", doc_type="visualization", id="system_bandwidth_line-char", body=doc) -if res['created']=="True": + index=".kibana", + doc_type="visualization", + id="system_bandwidth_line-char", + body=doc) +if res['created'] == "True": print("system_bandwidth_line-char visualization has created") else: print("system_bandwidth_line-char visualization has existed") -doc = { - "title": "system_bandwidth_terms_data", - "visState": "{\"title\":\"system_bandwidth_terms_data\",\"type\":\"table\",\"params\":{\"perPage\":80,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"throughput\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"tx_msg_size\",\"size\":200,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"5\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"rx_msg_size\",\"size\":200,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"6\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"mean_latency\",\"size\":5,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"7\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"local_cpu_util\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"_term\"}},{\"id\":\"8\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"remote_cpu_util\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } -} - -res = es.index( - index=".kibana", doc_type="visualization", id="system_bandwidth_terms_data", body=doc) -if res['created']=="True": +# Create monitoring data in table +with open(dashboard_dir + 'posca_system_bandwidth\ +_terms_data.json') as terms_char: + doc = json.load(terms_char) +res = es.index(index=".kibana", doc_type="visualization", + id="system_bandwidth_terms_data", body=doc) +if res['created'] == "True": print("system_bandwidth_terms_data visualization has created") else: print("system_bandwidth_terms_data visualization has existed") -doc = { - "title": "system_bandwidth_dashboard", - "hits": 0, - "description": "", - "panelsJSON": "[{\"id\":\"system_bandwidth_line-char\",\"type\":\"visualization\",\"panelIndex\":3,\"size_x\":7,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"system_bandwidth_line-date\",\"type\":\"visualization\",\"panelIndex\":4,\"size_x\":7,\"size_y\":4,\"col\":1,\"row\":5},{\"id\":\"system_bandwidth_terms_data\",\"type\":\"visualization\",\"panelIndex\":5,\"size_x\":5,\"size_y\":8,\"col\":8,\"row\":1}]", - "optionsJSON": "{\"darkTheme\":false}", - "uiStateJSON": "{}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}" - } -} - +# Create dashboard +with open(dashboard_dir + 'posca_system_bandwidth\ +_dashboard.json') as dashboard: + doc = json.load(dashboard) res = es.index( - index=".kibana", doc_type="dashboard", id="system_bandwidth_dashboard", body=doc) -if res['created']=="True": + index=".kibana", + doc_type="dashboard", + id="system_bandwidth_dashboard", + body=doc) +if res['created'] == "True": print("system_bandwidth dashboard has created") else: print("system_bandwidth dashboard has existed") diff --git a/testsuites/posca/testcase_script/common_script.py b/testsuites/posca/testcase_script/common_script.py index 2ca9f68b..a0114644 100644 --- a/testsuites/posca/testcase_script/common_script.py +++ b/testsuites/posca/testcase_script/common_script.py @@ -19,38 +19,38 @@ headers = {"Content-Type": "application/json"} def posca_tran_data(ES_ip, file_name): - p = sub.Popen(['curl', '-s', '-XPOST', "%s/_bulk" % ES_ip, - '--data-binary', "@" + file_name], stdout=sub.PIPE) - for line in iter(p.stdout.readline, b''): - ret_dict = json.loads(line) - if not ret_dict['errors']: - print("INFO: %6s lines no errors, total cost %d ms." - % (len(ret_dict['items']), ret_dict['took'])) - return len(ret_dict['items']) - else: - print("ERROR: %6s lines have errors, total cost %d ms." - % (len(ret_dict['items']), ret_dict['took'])) + p = sub.Popen(['curl', '-s', '-XPOST', "%s/_bulk" % ES_ip, + '--data-binary', "@" + file_name], stdout=sub.PIPE) + for line in iter(p.stdout.readline, b''): + ret_dict = json.loads(line) + if not ret_dict['errors']: + print("INFO: %6s lines no errors, total cost %d ms." + % (len(ret_dict['items']), ret_dict['took'])) + return len(ret_dict['items']) + else: + print("ERROR: %6s lines have errors, total cost %d ms." + % (len(ret_dict['items']), ret_dict['took'])) def posca_config_read(config_str, con_str, config): print("========== posca system bandwidth config read ===========") con_dic = {} - print("Configuration file is %s" %(config_str)) + print("Configuration file is %s" % (config_str)) idx = 0 with open(config_str, "rd") as cfgfile: config.readfp(cfgfile) while idx < len(con_str): con_dic[str(con_str[idx])] = \ - config.get("config", str(con_str[idx])) + config.get("config", str(con_str[idx])) idx += 1 with IPDB() as ip: GATEWAY_IP = ip.routes['default'].gateway if str(con_dic["test_ip"]) is "": - con_dic["test_ip"] = GATEWAY_IP+":8888" - print("test_ip is null get local ip is %s" %(con_dic["test_ip"])) + con_dic["test_ip"] = GATEWAY_IP + ":8888" + print("test_ip is null get local ip is %s" % (con_dic["test_ip"])) if con_dic["ES_ip"] is "": - con_dic["ES_ip"] = GATEWAY_IP+":9200" - print("ES_ip is null get local ip is %s" %(con_dic["ES_ip"])) + con_dic["ES_ip"] = GATEWAY_IP + ":9200" + print("ES_ip is null get local ip is %s" % (con_dic["ES_ip"])) return con_dic @@ -81,44 +81,46 @@ def posca_get_reply(con_dic, task_id, time_test=1): time.sleep(float(con_dic["test_time"])) reply_response = requests.get(reply_url) reply_data = json.loads(reply_response.text) - print("return data is %s" %(reply_data)) + print("return data is %s" % (reply_data)) if reply_data["status"] == 1: return(reply_data["result"][0]) if reply_data["status"] == 0: if time_test == 10: print("yardstick time out") sys.exit() - reply_result_data = posca_get_reply(con_dic, task_id, time_test=time_test+1) + reply_result_data = posca_get_reply( + con_dic, task_id, time_test=time_test + 1) return(reply_result_data) - posca_get_reply(con_dic, task_id, time_test=time_test+1) + posca_get_reply(con_dic, task_id, time_test=time_test + 1) if reply_data["status"] == 2: print("yardstick error exit") sys.exit() def posca_send_data(con_dic, test_config, file_config): - base_url = "http://%s/yardstick/testcases/samples/action" % (con_dic['test_ip']) - print("test ip addr is %s" %(base_url)) + base_url = "http://%s/yardstick/testcases/samples/action" % (con_dic[ + 'test_ip']) + print("test ip addr is %s" % (base_url)) test_dict = { - "action":"runTestCase", - "args":{ - "opts": { - "task-args": { - 'tx_msg_size': '%s' % str(test_config["tx_msg_size"]), - 'rx_msg_size': '%s' % str(test_config["rx_msg_size"]), - 'test_time': '%s' % str(int(con_dic["test_time"]) - 20), - 'host': 'node3.LF', - 'target': 'node4.LF' - } - }, - "testcase":"netperf_bottlenecks" - } + "action": "runTestCase", + "args": { + "opts": { + "task-args": { + 'tx_msg_size': '%s' % str(test_config["tx_msg_size"]), + 'rx_msg_size': '%s' % str(test_config["rx_msg_size"]), + 'test_time': '%s' % str(int(con_dic["test_time"]) - 20), + 'host': 'node3.LF', + 'target': 'node4.LF' + } + }, + "testcase": "netperf_bottlenecks" + } } reponse = requests.post( - base_url, data=json.dumps(test_dict), headers=headers) + base_url, data=json.dumps(test_dict), headers=headers) ask_data = json.loads(reponse.text) task_id = ask_data["result"] - print("yardstick task id is: %s" %(task_id)) + print("yardstick task id is: %s" % (task_id)) data_reply = posca_get_reply(con_dic, task_id) data_reply.update(test_config) posca_output_result(file_config, data_reply) @@ -128,11 +130,10 @@ def posca_send_data(con_dic, test_config, file_config): def posca_create_incluxdb(con_dic): base_url = "http://%s/yardstick/env/action" % (con_dic['test_ip']) test_dict = { - "action":"createInfluxDBContainer", + "action": "createInfluxDBContainer", } - reponse = requests.post( - base_url, data=json.dumps(test_dict), headers=headers) + requests.post( + base_url, data=json.dumps(test_dict), headers=headers) print("waiting for creating InfluxDB") time.sleep(30) print("Done, creating InflxDB Container") - diff --git a/testsuites/posca/testcase_script/posca_factor_cpu_burden.py b/testsuites/posca/testcase_script/posca_factor_cpu_burden.py index 273a24f9..58da8ca7 100644 --- a/testsuites/posca/testcase_script/posca_factor_cpu_burden.py +++ b/testsuites/posca/testcase_script/posca_factor_cpu_burden.py @@ -49,9 +49,6 @@ def posca_env_check(): def system_cpu_burden(test_id, data, file_config, con_dic): date_id = test_id print("test is is begin from %d" % test_id) - cur_role_result = 1 - pre_role_result = 1 - pre_reply = {} data_return = {} data_max = {} data_return["throughput"] = 1 @@ -59,14 +56,13 @@ def system_cpu_burden(test_id, data, file_config, con_dic): data_max["throughput"] = 1 for test_y in data["rx_pkt_sizes"]: test_config = { - "tx_msg_size": float(test_x), - "rx_msg_size": float(test_y), - } + "tx_msg_size": float(test_x), + "rx_msg_size": float(test_y), + } date_id = date_id + 1 file_config["test_id"] = date_id data_reply = common_script.posca_send_data( - con_dic, test_config, file_config) - bandwidth = data_reply["throughput"] + con_dic, test_config, file_config) if (data_max["remote_cpu_util"] > con_dic["cpu_load"]): return 1, data_reply if (data_max["local_cpu_util"] > con_dic["cpu_load"]): @@ -90,7 +86,7 @@ test_result/factor_system_system_bandwidth_%s.json" % (time_new) data["tx_pkt_sizes"] = tx_pkt_s_a print("######test package begin######") date_return, pkt_reply = system_cpu_burden( - test_con_id, data, file_config, con_dic) + test_con_id, data, file_config, con_dic) return True @@ -104,9 +100,9 @@ def main(): testcase_cfg = args.conf con_str = [ - 'test_ip', 'tool', 'test_time', 'protocol', - 'tx_pkt_sizes', 'rx_pkt_sizes', 'cpu_load', - 'latency', 'ES_ip', 'dashboard' + 'test_ip', 'tool', 'test_time', 'protocol', + 'tx_pkt_sizes', 'rx_pkt_sizes', 'cpu_load', + 'latency', 'ES_ip', 'dashboard' ] posca_env_check() starttime = datetime.datetime.now() @@ -122,8 +118,8 @@ system_bandwidth.py' print("\nBegin to establish dashboard.") sub_result = subprocess.Popen(pargs) sub_result.wait() - print("System Bandwidth testing time : %s" %(endtime - starttime)) + print("System Bandwidth testing time : %s" % (endtime - starttime)) time.sleep(5) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/testsuites/posca/testcase_script/posca_factor_rx_cache_size.py b/testsuites/posca/testcase_script/posca_factor_rx_cache_size.py index cfbd278e..97045a1d 100644 --- a/testsuites/posca/testcase_script/posca_factor_rx_cache_size.py +++ b/testsuites/posca/testcase_script/posca_factor_rx_cache_size.py @@ -110,7 +110,7 @@ def posca_run(con_dic): bandwidth, latency, cpu_load) if (cpu_load < con_dic['test_cpu_load\ ']) and (latency < con_dic['test_latency']): - if (abs(bandwidth_tmp-bandwidth)/bandwidth < 0.05): + if (abs(bandwidth_tmp - bandwidth) / bandwidth < 0.05): return True else: print("%s,%s") % (bandwidth, test_tx_cache_s_e) diff --git a/testsuites/posca/testcase_script/posca_factor_rx_pkt_size.py b/testsuites/posca/testcase_script/posca_factor_rx_pkt_size.py index 5de25949..2a119413 100644 --- a/testsuites/posca/testcase_script/posca_factor_rx_pkt_size.py +++ b/testsuites/posca/testcase_script/posca_factor_rx_pkt_size.py @@ -115,7 +115,7 @@ def posca_run(con_dic): bandwidth, latency, cpu_load) if (cpu_load < con_dic['test_cpu_load\ ']) and (latency < con_dic['test_latency']): - if (abs(bandwidth_tmp-bandwidth)/bandwidth < 0.05): + if (abs(bandwidth_tmp - bandwidth) / bandwidth < 0.05): print("%s,%s,%s,%s,%s,%s") % (test_rx_pkt_s_e, test_rx_cache_s_e, test_tx_cache_s_e, diff --git a/testsuites/posca/testcase_script/posca_factor_system_bandwidth.py b/testsuites/posca/testcase_script/posca_factor_system_bandwidth.py index 0b77b6b6..7a0fd27b 100644 --- a/testsuites/posca/testcase_script/posca_factor_system_bandwidth.py +++ b/testsuites/posca/testcase_script/posca_factor_system_bandwidth.py @@ -60,24 +60,24 @@ def system_pkt_bandwidth(test_id, data, file_config, con_dic): bandwidth_tmp = 1 for test_y in data["rx_pkt_sizes"]: test_config = { - "tx_msg_size": float(test_x), - "rx_msg_size": float(test_y), - } + "tx_msg_size": float(test_x), + "rx_msg_size": float(test_y), + } date_id = date_id + 1 file_config["test_id"] = date_id data_reply = common_script.posca_send_data( - con_dic, test_config, file_config) + con_dic, test_config, file_config) bandwidth = data_reply["throughput"] if (data_max["throughput"] < bandwidth): data_max = data_reply - if (abs(bandwidth_tmp - bandwidth)/bandwidth_tmp < 0.025): + if (abs(bandwidth_tmp - bandwidth) / bandwidth_tmp < 0.025): print(pre_reply) break else: pre_reply = data_reply bandwidth_tmp = bandwidth cur_role_result = pre_reply["throughput"] - if (abs(pre_role_result - cur_role_result)/pre_role_result < 0.025): + if (abs(pre_role_result - cur_role_result) / pre_role_result < 0.025): print("date_id is %d,package return at line 111\n" % date_id) # return data_return if data_return["throughput"] < data_max["throughput"]: @@ -107,8 +107,8 @@ test_result/factor_system_system_bandwidth_%s.json" % (time_new) print("######find system bandwidth######") print("rx_msg_size:%d tx_msg_size:%d\n" % (pkt_reply["rx_msg_size"], pkt_reply["tx_msg_size"])) - date_tran = common_script.posca_tran_data( - con_dic['ES_ip'], file_config["file_path"]) + common_script.posca_tran_data( + con_dic['ES_ip'], file_config["file_path"]) return True @@ -121,9 +121,9 @@ def main(): testcase_cfg = args.conf con_str = [ - 'test_ip', 'tool', 'test_time', 'protocol', - 'tx_pkt_sizes', 'rx_pkt_sizes', 'cpu_load', - 'latency', 'ES_ip', 'dashboard' + 'test_ip', 'tool', 'test_time', 'protocol', + 'tx_pkt_sizes', 'rx_pkt_sizes', 'cpu_load', + 'latency', 'ES_ip', 'dashboard' ] posca_env_check() starttime = datetime.datetime.now() @@ -139,9 +139,8 @@ system_bandwidth.py' print("\nBegin to establish dashboard.") sub_result = subprocess.Popen(pargs) sub_result.wait() - print("System Bandwidth testing time : %s" %(endtime - starttime)) + print("System Bandwidth testing time : %s" % (endtime - starttime)) time.sleep(5) if __name__ == '__main__': main() - diff --git a/testsuites/posca/testcase_script/posca_factor_tx_cache_size.py b/testsuites/posca/testcase_script/posca_factor_tx_cache_size.py index a8d1a485..b7d45065 100644 --- a/testsuites/posca/testcase_script/posca_factor_tx_cache_size.py +++ b/testsuites/posca/testcase_script/posca_factor_tx_cache_size.py @@ -106,10 +106,10 @@ def posca_run(con_dic): latency = test_result.find("latency") posca_output_result(time_new, test_pkt_s_e, test_rx_cache_s_e, bandwidth, latency, cpu_load) - if (abs(bandwidth-con_dic['test_throughput'])/con_dic['test_\ + if (abs(bandwidth - con_dic['test_throughput']) / con_dic['test_\ throughput'] > 0.05) and (latency < con_dic['test_\ latency']) and (cpu_load < con_dic['test_cpu_load']): - if (abs(bandwidth_tmp-bandwidth)/bandwidth < 0.05): + if (abs(bandwidth_tmp - bandwidth) / bandwidth < 0.05): return True else: print("%s,%s") % (bandwidth, test_rx_cache_s_e) diff --git a/testsuites/posca/testcase_script/posca_factor_tx_pkt_size.py b/testsuites/posca/testcase_script/posca_factor_tx_pkt_size.py index 47cc5e75..39dba135 100644 --- a/testsuites/posca/testcase_script/posca_factor_tx_pkt_size.py +++ b/testsuites/posca/testcase_script/posca_factor_tx_pkt_size.py @@ -112,10 +112,10 @@ def posca_run(con_dic): posca_output_result(time_new, test_rx_pkt_s_e, test_rx_cache_s_e, test_tx_cache_s_e, bandwidth, latency, cpu_load) - if (abs(bandwidth-con_dic['test_throughput'])/con_dic['test_\ + if (abs(bandwidth - con_dic['test_throughput']) / con_dic['test_\ throughput'] > 0.05) and (latency < con_dic['test_\ latency']) and (cpu_load < con_dic['test_cpu_load']): - if (abs(bandwidth_tmp-bandwidth)/bandwidth < 0.05): + if (abs(bandwidth_tmp - bandwidth) / bandwidth < 0.05): print("%s,%s,%s,%s,%s,%s") % (test_rx_pkt_s_e, test_rx_cache_s_e, test_tx_cache_s_e, -- cgit 1.2.3-korg