diff options
| -rw-r--r-- | config/config.yaml | 5 | ||||
| -rw-r--r-- | testsuites/posca/testcase_cfg/posca_factor_multistack_storage.yaml | 35 | ||||
| -rw-r--r-- | testsuites/posca/testcase_cfg/posca_factor_multistack_storage_parallel.yaml | 33 | ||||
| -rw-r--r-- | testsuites/posca/testcase_script/posca_factor_multistack_storage.py | 236 | ||||
| -rw-r--r-- | testsuites/posca/testcase_script/posca_factor_multistack_storage_parallel.py | 164 | ||||
| -rw-r--r-- | testsuites/posca/testcase_script/posca_factor_ping.py | 18 | ||||
| -rw-r--r-- | testsuites/posca/testcase_script/posca_feature_testpmd_scale_up.py | 5 | ||||
| -rw-r--r-- | testsuites/posca/testcase_script/posca_feature_vnf_scale_out.py | 10 | ||||
| -rw-r--r-- | testsuites/run_testsuite.py | 19 | ||||
| -rw-r--r-- | utils/env_prepare/quota_prepare.py | 27 | ||||
| -rw-r--r-- | utils/env_prepare/stack_prepare.py | 37 | ||||
| -rw-r--r-- | utils/infra_setup/heat/manager.py | 2 | ||||
| -rw-r--r-- | utils/infra_setup/runner/yardstick.py | 15 | ||||
| -rw-r--r-- | utils/parser.py | 5 |
14 files changed, 581 insertions, 30 deletions
diff --git a/config/config.yaml b/config/config.yaml index 76e42b30..f85599c1 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -13,4 +13,7 @@ common_config: pod_info: '/tmp/pod.yaml' yardstick_rc_dir: '/etc/yardstick/openstack.creds' releng_dir: '/home/opnfv/releng' - fetch_os_file: 'utils/fetch_os_creds.sh'
\ No newline at end of file + fetch_os_file: 'utils/fetch_os_creds.sh' + yardstick_image_dir: '/tmp/yardstick.img' + image_url: 'http://artifacts.opnfv.org/yardstick/image/yardstick-nsb-image.img' + yardstick_image_name: 'bottlenecks_yardstick' diff --git a/testsuites/posca/testcase_cfg/posca_factor_multistack_storage.yaml b/testsuites/posca/testcase_cfg/posca_factor_multistack_storage.yaml new file mode 100644 index 00000000..e2f48438 --- /dev/null +++ b/testsuites/posca/testcase_cfg/posca_factor_multistack_storage.yaml @@ -0,0 +1,35 @@ +############################################################################## +# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +load_manager: + scenarios: + tool: fio + test_times: 10 + #this "rw" value could be write read rw rr or randrw + rw: 'randrw' + bs: '4k' + size: '20g' + rwmixwrite: '50' + num_thread: 1, 2 + num_stack: 1 + volume_num: '1' + num_jobs: '1' + direct: '1' + volume_size: 50 + + runners: + stack_create: yardstick + flavor: + yardstick_test_dir: "samples" + yardstick_testcase: "storage_bottlenecks" + +contexts: + dashboard: "Bottlenecks-ELK" + yardstick: "Bottlenecks-Yardstick" + yardstick_envpre: True diff --git a/testsuites/posca/testcase_cfg/posca_factor_multistack_storage_parallel.yaml b/testsuites/posca/testcase_cfg/posca_factor_multistack_storage_parallel.yaml new file mode 100644 index 00000000..b55b826c --- /dev/null +++ b/testsuites/posca/testcase_cfg/posca_factor_multistack_storage_parallel.yaml @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +load_manager: + scenarios: + tool: fio + # for this option we provide " write, read, rw, rr" + rw: "randrw" + bs: "4k" + size: "20g" + rwmixwrite: "50" + num_stack: 1, 3 + volume_num: "1" + num_jobs: "1" + direct: "1" + volume_size: 50 + + runners: + stack_create: yardstick + flavor: + yardstick_test_dir: "samples" + yardstick_testcase: "storage_bottlenecks" + +contexts: + dashboard: "Bottlenecks-ELK" + yardstick: "Bottlenecks-Yardstick" + yardstick_envpre: True diff --git a/testsuites/posca/testcase_script/posca_factor_multistack_storage.py b/testsuites/posca/testcase_script/posca_factor_multistack_storage.py new file mode 100644 index 00000000..34ee225c --- /dev/null +++ b/testsuites/posca/testcase_script/posca_factor_multistack_storage.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +'''This file realize the function of run posca multistack storage stress test +This file contain several part: +First is create a script to realize several threading run''' + +import utils.logger as log +import uuid +import json +import os +import time +import threading +import datetime +import Queue +from utils.parser import Parser as conf_parser +import utils.env_prepare.quota_prepare as quota_prepare +import utils.env_prepare.stack_prepare as stack_prepare +import utils.infra_setup.runner.yardstick as yardstick_task + +import utils.infra_setup.runner.docker_env as docker_env + +# -------------------------------------------------- +# logging configuration +# -------------------------------------------------- +LOG = log.Logger(__name__).getLogger() + +test_dict = { + "action": "runTestCase", + "args": { + "opts": { + "task-args": {} + }, + "testcase": "multistack_storage_bottlenecks" + } +} +testfile = os.path.basename(__file__) +testcase, file_format = os.path.splitext(testfile) +cidr = "/home/opnfv/repos/yardstick/samples/storage_bottlenecks.yaml" +runner_DEBUG = True +q = Queue.Queue() +final_result = Queue.Queue() + + +def env_pre(test_config): + test_yardstick = False + if "yardstick" in test_config["contexts"].keys(): + test_yardstick = True + stack_prepare._prepare_env_daemon(test_yardstick) + quota_prepare.quota_env_prepare() + LOG.info("yardstick environment prepare!") + if(test_config["contexts"]['yardstick_envpre']): + stdout = yardstick_task.yardstick_image_prepare() + LOG.debug(stdout) + + +def testcase_parser(out_file="yardstick.out", **parameter_info): + cmd = yardstick_task.yardstick_command_parser(debug=runner_DEBUG, + cidr=cidr, + outfile=out_file, + parameter=parameter_info) + return cmd + + +def do_test(test_config): + out_file = ("/tmp/yardstick_" + str(uuid.uuid4()) + ".out") + yardstick_container = docker_env.yardstick_info['container'] + cmd = testcase_parser(out_file=out_file, **test_config) + print(cmd) + stdout = docker_env.docker_exec_cmd(yardstick_container, cmd) + LOG.info(stdout) + loop_value = 0 + while loop_value < 60: + time.sleep(2) + loop_value = loop_value + 1 + with open(out_file) as f: + data = json.load(f) + if data["result"]["criteria"] == "PASS": + LOG.info("yardstick run success") + LOG.info("%s" % data["result"]["testcases"]) + break + else: + LOG.error("yardstick error exit") + break + + save_data = final_config_to_result(test_config, data) + return save_data + + +def config_to_result(num, out_num, during_date): + testdata = {} + test_result = {} + final_data = {} + + final_data["read_iops"] = 0 + final_data["read_bw"] = 0 + final_data["read_lat"] = 0 + final_data["write_iops"] = 0 + final_data["write_bw"] = 0 + final_data["write_lat"] = 0 + + test_result["number_of_stacks"] = float(num) + test_result["success_times"] = out_num + test_result["success_rate"] = out_num / num + test_result["duration_time"] = during_date + testdata["data_body"] = test_result + testdata["testcase"] = testcase + + while not final_result.empty(): + data = final_result.get() + final_data["read_iops"] += data["read_iops"] + final_data["read_bw"] += data["read_bw"] + if final_data["read_lat"] is 0: + final_data["read_lat"] = data["read_lat"] + else: + final_data["read_lat"] += data["read_lat"] + final_data["read_lat"] = final_data["read_lat"]/2 + final_data["write_iops"] += data["write_iops"] + final_data["write_bw"] += data["read_iops"] + if final_data["write_lat"] is 0: + final_data["write_lat"] = data["write_lat"] + else: + final_data["write_lat"] += data["write_lat"] + final_data["write_lat"] = final_data["write_lat"]/2 + + testdata["test_value"] = final_data + LOG.info("Final testdata is %s" % testdata) + return testdata + + +def final_config_to_result(test_config, test_result): + out_data = test_result["result"]["testcases"] + test_data = out_data["storage_bottlenecks"]["tc_data"] + testdata = {} + testdata["read_iops"] = 0 + testdata["read_bw"] = 0 + testdata["read_lat"] = 0 + testdata["write_iops"] = 0 + testdata["write_bw"] = 0 + testdata["write_lat"] = 0 + print(testdata["read_iops"]) + for result in test_data: + testdata["read_iops"] += result["data"]["read_iops"] + testdata["read_bw"] += result["data"]["read_bw"] + if testdata["read_lat"] is 0: + testdata["read_lat"] = result["data"]["read_lat"] + else: + testdata["read_lat"] += result["data"]["read_lat"] + testdata["read_lat"] = testdata["read_lat"]/2 + testdata["write_iops"] += result["data"]["write_iops"] + testdata["write_bw"] += result["data"]["write_bw"] + if testdata["write_lat"] is 0: + testdata["write_lat"] = result["data"]["write_lat"] + else: + testdata["write_lat"] += result["data"]["write_lat"] + testdata["write_lat"] = testdata["write_lat"]/2 + final_result.put(testdata) + q.put(1) + return testdata + + +def func_run(con_dic): + test_date = do_test(con_dic) + return test_date + + +def run(test_config): + con_dic = test_config["load_manager"] + scenarios_conf = con_dic["scenarios"] + + if test_config["contexts"]["yardstick_ip"] is None: + con_dic["contexts"]["yardstick_ip"] =\ + conf_parser.ip_parser("yardstick_test_ip") + + env_pre(test_config) + LOG.info("yardstick environment prepare done!") + + stack_num = scenarios_conf["num_stack"] + test_num = conf_parser.str_to_list(scenarios_conf["num_thread"]) + rw = scenarios_conf["rw"] + bs = scenarios_conf["bs"] + size = scenarios_conf["size"] + rwmixwrite = scenarios_conf["rwmixwrite"] + numjobs = scenarios_conf["num_jobs"] + direct = scenarios_conf["direct"] + volume_num = scenarios_conf["volume_num"] + volume_size = scenarios_conf["volume_size"] + + for value in test_num: + result = [] + out_num = 0 + num = int(value) + # pool = multiprocessing.Pool(processes=num) + threadings = [] + LOG.info("begin to run %s thread" % num) + + starttime = datetime.datetime.now() + + for i in xrange(0, num): + case_config = {"stack_num": int(stack_num), + "volume_num": volume_num, + "rw": rw, + "bs": bs, + "size": size, + "rwmixwrite": rwmixwrite, + "numjobs": numjobs, + "direct": direct, + "volume_size": int(volume_size)} + tmp_thread = threading.Thread(target=func_run, args=(case_config,)) + threadings.append(tmp_thread) + tmp_thread.start() + + for one_thread in threadings: + one_thread.join() + while not q.empty(): + result.append(q.get()) + for item in result: + out_num = out_num + float(item) + + print(result) + + endtime = datetime.datetime.now() + LOG.info("%s thread success %d times" % (num, out_num)) + during_date = (endtime - starttime).seconds + + data_reply = config_to_result(num, out_num, during_date) + conf_parser.result_to_file(data_reply, test_config["out_file"]) + + LOG.info('END POSCA stress multistack storage test') + return data_reply diff --git a/testsuites/posca/testcase_script/posca_factor_multistack_storage_parallel.py b/testsuites/posca/testcase_script/posca_factor_multistack_storage_parallel.py new file mode 100644 index 00000000..8c623d41 --- /dev/null +++ b/testsuites/posca/testcase_script/posca_factor_multistack_storage_parallel.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +'''This file realize the function of run posca multistack storage stress test +This file contain several part: +First is create a script to realize several threading run''' + +import utils.logger as log +import uuid +import json +import os +import time +from utils.parser import Parser as conf_parser +import utils.env_prepare.quota_prepare as quota_prepare +import utils.env_prepare.stack_prepare as stack_prepare +import utils.infra_setup.runner.yardstick as yardstick_task + +import utils.infra_setup.runner.docker_env as docker_env + +# -------------------------------------------------- +# logging configuration +# -------------------------------------------------- +LOG = log.Logger(__name__).getLogger() + +test_dict = { + "action": "runTestCase", + "args": { + "opts": { + "task-args": {} + }, + "testcase": "multistack_storage_bottlenecks_parallel" + } +} +testfile = os.path.basename(__file__) +testcase, file_format = os.path.splitext(testfile) +cidr = "/home/opnfv/repos/yardstick/samples/storage_bottlenecks.yaml" +runner_DEBUG = True + + +def env_pre(test_config): + test_yardstick = False + if "yardstick" in test_config["contexts"].keys(): + test_yardstick = True + stack_prepare._prepare_env_daemon(test_yardstick) + quota_prepare.quota_env_prepare() + if(test_config["contexts"]['yardstick_envpre']): + LOG.info("yardstick environment prepare!") + stdout = yardstick_task.yardstick_image_prepare() + LOG.debug(stdout) + + +def testcase_parser(out_file="yardstick.out", **parameter_info): + cmd = yardstick_task.yardstick_command_parser(debug=runner_DEBUG, + cidr=cidr, + outfile=out_file, + parameter=parameter_info) + return cmd + + +def do_test(test_config): + out_file = ("/tmp/yardstick_" + str(uuid.uuid4()) + ".out") + yardstick_container = docker_env.yardstick_info['container'] + cmd = testcase_parser(out_file=out_file, **test_config) + print(cmd) + stdout = docker_env.docker_exec_cmd(yardstick_container, cmd) + LOG.info(stdout) + loop_value = 0 + while loop_value < 60: + time.sleep(2) + loop_value = loop_value + 1 + with open(out_file) as f: + data = json.load(f) + if data["result"]["criteria"] == "PASS": + LOG.info("yardstick run success") + LOG.info("%s" % data["result"]["testcases"]) + break + else: + LOG.error("yardstick error exit") + break + + save_data = config_to_result(test_config, data) + LOG.info(save_data) + return save_data + + +def config_to_result(test_config, test_result): + print(test_result) + out_data = test_result["result"]["testcases"] + test_data = out_data["storage_bottlenecks"]["tc_data"] + testdata = {} + testdata["read_iops"] = 0 + testdata["read_bw"] = 0 + testdata["read_lat"] = 0 + testdata["write_iops"] = 0 + testdata["write_bw"] = 0 + testdata["write_lat"] = 0 + print(testdata["read_iops"]) + for result in test_data: + testdata["read_iops"] += result["data"]["read_iops"] + testdata["read_bw"] += result["data"]["read_bw"] + if testdata["read_lat"] is 0: + testdata["read_lat"] = result["data"]["read_lat"] + else: + testdata["read_lat"] += result["data"]["read_lat"] + testdata["read_lat"] = testdata["read_lat"]/2 + testdata["write_iops"] += result["data"]["write_iops"] + testdata["write_bw"] += result["data"]["write_bw"] + if testdata["write_lat"] is 0: + testdata["write_lat"] = result["data"]["write_lat"] + else: + testdata["write_lat"] += result["data"]["write_lat"] + testdata["write_lat"] = testdata["write_lat"]/2 + return testdata + + +def run(test_config): + con_dic = test_config["load_manager"] + scenarios_conf = con_dic["scenarios"] + + if test_config["contexts"]["yardstick_ip"] is None: + con_dic["contexts"]["yardstick_ip"] =\ + conf_parser.ip_parser("yardstick_test_ip") + + env_pre(test_config) + LOG.info("yardstick environment prepare done!") + + test_num = conf_parser.str_to_list(scenarios_conf["num_stack"]) + rw = scenarios_conf["rw"] + bs = scenarios_conf["bs"] + size = scenarios_conf["size"] + rwmixwrite = scenarios_conf["rwmixwrite"] + numjobs = scenarios_conf["num_jobs"] + direct = scenarios_conf["direct"] + volume_num = scenarios_conf["volume_num"] + volume_size = scenarios_conf["volume_size"] + + result = [] + + for value in test_num: + case_config = {"stack_num": int(value), + "volume_num": volume_num, + "rw": rw, + "bs": bs, + "size": size, + "rwmixwrite": rwmixwrite, + "numjobs": numjobs, + "direct": direct, + "volume_size": int(volume_size)} + data_reply = do_test(case_config) + result.append(data_reply) + + LOG.info("%s stack successful run" % (value)) + + conf_parser.result_to_file(data_reply, test_config["out_file"]) + + LOG.info('END POSCA stress multistack storage parallel testcase') + LOG.info("The result data is %s", result) + return result diff --git a/testsuites/posca/testcase_script/posca_factor_ping.py b/testsuites/posca/testcase_script/posca_factor_ping.py index 3a2277cf..4ee43964 100644 --- a/testsuites/posca/testcase_script/posca_factor_ping.py +++ b/testsuites/posca/testcase_script/posca_factor_ping.py @@ -23,6 +23,7 @@ import Queue from utils.parser import Parser as conf_parser import utils.env_prepare.quota_prepare as quota_prepare import utils.env_prepare.stack_prepare as stack_prepare +import utils.infra_setup.runner.yardstick as runner_yardstick import testsuites.posca.testcase_dashboard.posca_stress_ping as DashBoard import utils.infra_setup.runner.docker_env as docker_env @@ -42,6 +43,8 @@ test_dict = { } testfile = os.path.basename(__file__) testcase, file_format = os.path.splitext(testfile) +cidr = "/home/opnfv/repos/yardstick/samples/ping_bottlenecks.yaml" +runner_DEBUG = True q = Queue.Queue() @@ -52,20 +55,21 @@ def env_pre(test_config): test_yardstick = True stack_prepare._prepare_env_daemon(test_yardstick) quota_prepare.quota_env_prepare() - cmd = ('yardstick env prepare') LOG.info("yardstick environment prepare!") if(test_config["contexts"]['yardstick_envpre']): - yardstick_container = docker_env.yardstick_info['container'] - stdout = docker_env.docker_exec_cmd(yardstick_container, cmd) + stdout = runner_yardstick.yardstick_image_prepare() LOG.debug(stdout) def do_test(): func_name = sys._getframe().f_code.co_name out_file = ("/tmp/yardstick_" + str(uuid.uuid4()) + ".out") + parameter_info = {} yardstick_container = docker_env.yardstick_info['container'] - cmd = ('yardstick task start /home/opnfv/repos/yardstick/' - 'samples/ping_bottlenecks.yaml --output-file ' + out_file) + cmd = runner_yardstick.yardstick_command_parser(debug=runner_DEBUG, + cidr=cidr, + outfile=out_file, + parameter=parameter_info) stdout = docker_env.docker_exec_cmd(yardstick_container, cmd) LOG.info(stdout) out_value = 0 @@ -75,11 +79,11 @@ def do_test(): loop_value = loop_value + 1 with open(out_file) as f: data = json.load(f) - if data["status"] == 1: + if data["result"]["criteria"] == "PASS": LOG.info("yardstick run success") out_value = 1 break - elif data["status"] == 2: + else: LOG.error("yardstick error exit") out_value = 0 break diff --git a/testsuites/posca/testcase_script/posca_feature_testpmd_scale_up.py b/testsuites/posca/testcase_script/posca_feature_testpmd_scale_up.py index 830ff73f..08c4cbe9 100644 --- a/testsuites/posca/testcase_script/posca_feature_testpmd_scale_up.py +++ b/testsuites/posca/testcase_script/posca_feature_testpmd_scale_up.py @@ -81,14 +81,10 @@ def do_test(test_config, Use_Dashboard, context_conf): elif data["status"] == 2: LOG.error("yardstick error exit") exit() - # data = json.load(output) save_data = config_to_result(test_config, data) if Use_Dashboard is True: print("use dashboard") - # DashBoard.dashboard_send_data(context_conf, save_data) - - # return save_data["data_body"] return save_data @@ -108,7 +104,6 @@ def run(test_config): conf_parser.ip_parser("dashboard") LOG.info("Create Dashboard data") Use_Dashboard = True - # DashBoard.dashboard_system_bandwidth(test_config["contexts"]) cpus = conf_parser.str_to_list(scenarios_conf["cpus"]) mems = conf_parser.str_to_list(scenarios_conf["mems"]) diff --git a/testsuites/posca/testcase_script/posca_feature_vnf_scale_out.py b/testsuites/posca/testcase_script/posca_feature_vnf_scale_out.py index 6d53515f..417cf2b9 100644 --- a/testsuites/posca/testcase_script/posca_feature_vnf_scale_out.py +++ b/testsuites/posca/testcase_script/posca_feature_vnf_scale_out.py @@ -46,12 +46,9 @@ def env_pre(test_config): print(test_yardstick) stack_prepare._prepare_env_daemon(test_yardstick) quota_prepare.quota_env_prepare() - cmd = ('yardstick env prepare') LOG.info("yardstick environment prepare!") - print docker_env.yardstick_info['container'] if(test_config["contexts"]['yardstick_envpre']): - yardstick_container = docker_env.yardstick_info['container'] - stdout = docker_env.docker_exec_cmd(yardstick_container, cmd) + stdout = yardstick_task.yardstick_image_prepare() LOG.debug(stdout) @@ -99,16 +96,11 @@ def do_test(test_config, Use_Dashboard, context_conf): elif data["status"] == 2: LOG.error("yardstick error exit") exit() - # data = json.load(output) save_data = config_to_result(test_config, data) - print("^^^^^^^^^^^^^^^^^^^^^^^^^") - print save_data if Use_Dashboard is True: print("use dashboard") - # DashBoard.dashboard_send_data(context_conf, save_data) - # return save_data["data_body"] return save_data diff --git a/testsuites/run_testsuite.py b/testsuites/run_testsuite.py index 2e82b205..0f03d3e7 100644 --- a/testsuites/run_testsuite.py +++ b/testsuites/run_testsuite.py @@ -36,6 +36,15 @@ LOG = log.Logger(__name__).getLogger() # ------------------------------------------------------ +def copy_hosts_file(): + LOG.info("Begin copying hosts file to Bottlenecks-Yardstick") + os.system('cp /etc/hosts /tmp/hosts') + yardstick_docker = docker_env.docker_find('Bottlenecks-Yardstick') + cp_cmd = 'sudo cp -f /tmp/hosts /etc/hosts' + docker_env.docker_exec_cmd(yardstick_docker, cp_cmd) + LOG.info("Done with copying hosts file to Bottlenecks-Yardstick") + + def posca_testcase_run(testsuite, testcase_script, test_config): module_string = "testsuites.%s.testcase_script.%s" % (testsuite, @@ -107,10 +116,12 @@ def testsuite_run(test_level, test_name, REPORT="False"): conf_parser.Parser.testcase_out_dir(testcase) start_date = datetime.datetime.now() docker_env_prepare(config[testcase]) - try: - posca_testcase_run(tester_parser[0], testcase, config[testcase]) - except Exception, e: - LOG.warning('e.message:\t%s', e.message) + copy_hosts_file() + #try: + # posca_testcase_run(tester_parser[0], testcase, config[testcase]) + #except Exception, e: + # LOG.warning('e.message:\t%s', e.message) + posca_testcase_run(tester_parser[0], testcase, config[testcase]) stop_date = datetime.datetime.now() LOG.info("End of %s testcase in POSCA testsuite", testcase) criteria = "FAIL" diff --git a/utils/env_prepare/quota_prepare.py b/utils/env_prepare/quota_prepare.py index 267e70ab..8dcdf3d6 100644 --- a/utils/env_prepare/quota_prepare.py +++ b/utils/env_prepare/quota_prepare.py @@ -42,14 +42,37 @@ nova_quota = {"ram": -1, "injected_file_path_bytes": -1} +def check_https_enabled(): + LOG.debug("Check if https is enabled in OpenStack") + os_auth_url = os.getenv('OS_AUTH_URL') + if os_auth_url.startswith('https'): + LOG.debug("https is enabled") + return True + LOG.debug("https is not enabled") + return False + + def quota_env_prepare(): + https_enabled = check_https_enabled() + insecure_option = '' + insecure = os.getenv('OS_INSECURE',) + if https_enabled: + LOG.info("https is enabled") + if insecure: + if insecure.lower() == "true": + insecure_option = ' --insecure ' + else: + LOG.warn("Env variable OS_INSECURE is {}: if https + no " + "credential used, it should be set as True." + .format(insecure)) + tenant_name = os.getenv("OS_TENANT_NAME") - cmd = ("openstack project list | grep " + + cmd = ("openstack {} project list | grep ".format(insecure_option) + tenant_name + " | awk '{print $2}'") result = commands.getstatusoutput(cmd) - if result[0] == 0: + if result[0] == 0 and 'exception' not in result[1]: LOG.info("Get %s project id is %s" % (tenant_name, result[1])) else: LOG.error("can't get openstack project id") diff --git a/utils/env_prepare/stack_prepare.py b/utils/env_prepare/stack_prepare.py index 25c2a29b..6b9bc510 100644 --- a/utils/env_prepare/stack_prepare.py +++ b/utils/env_prepare/stack_prepare.py @@ -14,6 +14,7 @@ from utils.logger import Logger from utils.parser import Parser as config import utils.infra_setup.heat.manager as utils import utils.infra_setup.runner.docker_env as docker_env +import utils.infra_setup.heat.manager as client_manager LOG = Logger(__name__).getLogger() @@ -73,7 +74,14 @@ def _source_file(rc_file): p = subprocess.Popen(". %s; env" % rc_file, stdout=subprocess.PIPE, shell=True) output = p.communicate()[0] - env = dict((line.split('=', 1) for line in output.splitlines())) + output_lines = output.splitlines() + env = list() + for line in output_lines: + if '=' in line: + env.append(tuple(line.split('=', 1))) + + env = dict(env) +# env = dict((line.split('=', 1) for line in output_lines)) os.environ.update(env) return env @@ -93,3 +101,30 @@ def _append_external_network(rc_file): except OSError as e: if e.errno != errno.EEXIST: raise + + +def prepare_image(image_name, image_dir): + glance_client = client_manager._get_glance_client() + if not os.path.isfile(image_dir): + LOG.error("Error: file %s does not exist.", image_dir) + return None + try: + images = glance_client.images.list() + image_id = next((i.id for i in images if i.name == image_name), None) + if image_id is not None: + LOG.info("Image %s already exists.", image_name) + else: + LOG.info("Creating image '%s' from '%s'...", image_name, image_dir) + + image = glance_client.images.create( + name=image_name, visibility="public", disk_format="qcow2", + container_format="bare") + image_id = image.id + with open(image_dir) as image_data: + glance_client.images.upload(image_id, image_data) + return image_id + except Exception: # pylint: disable=broad-except + LOG.error( + "Error [create_glance_image(glance_client, '%s', '%s')]", + image_name, image_dir) + return None diff --git a/utils/infra_setup/heat/manager.py b/utils/infra_setup/heat/manager.py index 5c181ae6..1a360b78 100644 --- a/utils/infra_setup/heat/manager.py +++ b/utils/infra_setup/heat/manager.py @@ -35,7 +35,7 @@ def _get_neutron_client(): return neutron_client -def stack_create_images( +def create_images( imagefile=None, image_name="bottlenecks_image"): print "========== Create image in OS ==========" diff --git a/utils/infra_setup/runner/yardstick.py b/utils/infra_setup/runner/yardstick.py index 559b9c10..616bcc52 100644 --- a/utils/infra_setup/runner/yardstick.py +++ b/utils/infra_setup/runner/yardstick.py @@ -15,15 +15,27 @@ At present, This file contain the following function: 4.how the process of task.''' import sys +import os import time import requests import json +import urllib import utils.logger as logger +from utils.parser import Parser as config +import utils.env_prepare.stack_prepare as env headers = {"Content-Type": "application/json"} LOG = logger.Logger(__name__).getLogger() +def yardstick_image_prepare(): + if not os.path.exists(config.bottlenecks_config["yardstick_image_dir"]): + urllib.urlretrieve(config.bottlenecks_config["image_url"], + config.bottlenecks_config["yardstick_image_dir"]) + env.prepare_image(config.bottlenecks_config["yardstick_image_name"], + config.bottlenecks_config["yardstick_image_dir"]) + + def yardstick_command_parser(debug, cidr, outfile, parameter): cmd = "yardstick" if debug: @@ -31,6 +43,9 @@ def yardstick_command_parser(debug, cidr, outfile, parameter): cmd += " task start " cmd += str(cidr) cmd += " --output-file " + outfile + image_name = config.bottlenecks_config["yardstick_image_name"] + parameter["image_name"] = image_name + print parameter if parameter is not None: cmd += " --task-args " + '"' + str(parameter) + '"' return cmd diff --git a/utils/parser.py b/utils/parser.py index b46a3b91..c0c10721 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -45,6 +45,11 @@ class Parser(): cls.bottlenecks_config["pod_info"] = common_config['pod_info'] cls.bottlenecks_config["yardstick_rc_dir"] = \ common_config['yardstick_rc_dir'] + cls.bottlenecks_config["yardstick_image_dir"] = \ + common_config['yardstick_image_dir'] + cls.bottlenecks_config["image_url"] = common_config['image_url'] + cls.bottlenecks_config["yardstick_image_name"] = \ + common_config['yardstick_image_name'] cls.config_dir_check(cls.bottlenecks_config["log_dir"]) @classmethod |