diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/dashboard/process_data.py | 41 | ||||
| -rwxr-xr-x | utils/dashboard/rubbos_collector.py | 22 | ||||
| -rwxr-xr-x | utils/dashboard/uploader.py | 8 | ||||
| -rw-r--r-- | utils/dispatcher/func.py | 2 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/common.py | 38 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/consts/files.py | 1 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/manager.py | 12 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/template.py | 5 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/tests/generate_template_test.py | 9 |
9 files changed, 88 insertions, 50 deletions
diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py index d4b2ccfb..e71a1e82 100644 --- a/utils/dashboard/process_data.py +++ b/utils/dashboard/process_data.py @@ -13,6 +13,7 @@ import sys from rubbos_collector import RubbosCollector from uploader import Uploader + def printUsage(): print "Usage: python process_data.py required_params(**) optional_params([])" print " ** -i|--input input_data_dir" @@ -21,12 +22,14 @@ def printUsage(): print " [] -o|--output output_file" print " [] -u|--upload yes|no" + def process(input_dir, suite_name): result = dict() if suite_name == "rubbos": result = RubbosCollector().collect_data(input_dir) return result + def writeResult(output_file, result): f = open(output_file, "w") if isinstance(result, list): @@ -34,41 +37,43 @@ def writeResult(output_file, result): f.write(str(elem) + "\n") f.close() + def uploadResult(conf, suite_name, result): Uploader(conf).upload_result(suite_name, result) + def main(): if len(sys.argv) < 7 or len(sys.argv) % 2 == 0: printUsage() - exit (1) + exit(1) i = 1 params = dict() while (i < len(sys.argv)): - if sys.argv[i]=="-i" or sys.argv[i]=="--input": - params["input"] = sys.argv[i+1] - if sys.argv[i]=="-s" or sys.argv[i]=="--suite": - params["suite"] = sys.argv[i+1] - if sys.argv[i]=="-c" or sys.argv[i]=="--conf": - params["conf"] = sys.argv[i+1] - if sys.argv[i]=="-o" or sys.argv[i]=="--output": - params["output"] = sys.argv[i+1] - if sys.argv[i]=="-u" or sys.argv[i]=="--upload": - params["upload"] = sys.argv[i+1] - i = i+2 - if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")): + if sys.argv[i] == "-i" or sys.argv[i] == "--input": + params["input"] = sys.argv[i + 1] + if sys.argv[i] == "-s" or sys.argv[i] == "--suite": + params["suite"] = sys.argv[i + 1] + if sys.argv[i] == "-c" or sys.argv[i] == "--conf": + params["conf"] = sys.argv[i + 1] + if sys.argv[i] == "-o" or sys.argv[i] == "--output": + params["output"] = sys.argv[i + 1] + if sys.argv[i] == "-u" or sys.argv[i] == "--upload": + params["upload"] = sys.argv[i + 1] + i = i + 2 + if not("input" in params and "suite" in params and "conf" in params): print "Lack some required parameters." - exit (1) + exit(1) result = process(params["input"], params["suite"]) print "Results:" for elem in result: print elem - if params.has_key("output"): - writeResult(params["output"],result) + if "output" in params: + writeResult(params["output"], result) - if params.has_key("upload") and params["upload"].lower()=="yes": + if "upload" in params and params["upload"].lower() == "yes": uploadResult(params["conf"], params["suite"], result) -if __name__=="__main__": +if __name__ == "__main__": main() diff --git a/utils/dashboard/rubbos_collector.py b/utils/dashboard/rubbos_collector.py index c9851739..d9f86032 100755 --- a/utils/dashboard/rubbos_collector.py +++ b/utils/dashboard/rubbos_collector.py @@ -10,16 +10,18 @@ import subprocess as subp + def exec_shell(cmd): - out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate() + out, err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate() return out.strip() + def get_onetime_data(dir_name): cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name client_node_num = int(exec_shell(cmd)) cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name each_client_num = int(exec_shell(cmd)) - total_client = (client_node_num+1) * each_client_num + total_client = (client_node_num + 1) * each_client_num cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name throughput = int(exec_shell(cmd)) @@ -39,11 +41,15 @@ class RubbosCollector(object): pass def collect_data(self, data_home): - cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home + cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home result = [] for subdir in exec_shell(cmd).split('\n'): - total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir) - result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request}) - result.sort(key=lambda x:x['client']) - - return result; + total_client, throughput, request, error_request = get_onetime_data( + data_home + '/' + subdir) + result.append({'client': total_client, + 'throughput': throughput, + 'request': request, + 'error_request': error_request}) + result.sort(key=lambda x: x['client']) + + return result diff --git a/utils/dashboard/uploader.py b/utils/dashboard/uploader.py index 07862fed..a4686560 100755 --- a/utils/dashboard/uploader.py +++ b/utils/dashboard/uploader.py @@ -29,7 +29,6 @@ class Uploader(object): self.result['version'] = dashboard_conf['version'] self.target = dashboard_conf['target'] - def upload_result(self, case_name, raw_data): if self.target == '': print('No target was set, so no data will be posted.') @@ -43,7 +42,9 @@ class Uploader(object): data=json.dumps(self.result), headers=self.headers, timeout=self.timeout) - print('Test result posting finished with status code %d.' % res.status_code) + print( + 'Test result posting finished with status code %d.' % + res.status_code) except Exception as err: print ('Failed to record result data: %s', err) @@ -55,10 +56,9 @@ def _test(): print ("no argumens input!!") exit(1) - with open(sys.argv[1],'r') as stream: + with open(sys.argv[1], 'r') as stream: data = json.load(stream) Uploader().upload_result(data) if __name__ == "__main__": _test() - diff --git a/utils/dispatcher/func.py b/utils/dispatcher/func.py index 71830bf2..cb907e5e 100644 --- a/utils/dispatcher/func.py +++ b/utils/dispatcher/func.py @@ -1,4 +1,4 @@ -#Copyright 2013: Mirantis Inc. +# Copyright 2013: Mirantis Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/utils/infra_setup/heat/common.py b/utils/infra_setup/heat/common.py index 24de893f..c4a78249 100755 --- a/utils/infra_setup/heat/common.py +++ b/utils/infra_setup/heat/common.py @@ -34,6 +34,7 @@ TEMPLATE_EXTENSION = None # Initialization and Input 'heat_templates/'validation # ------------------------------------------------------ + def init(api=False): global BASE_DIR # BASE_DIR = os.getcwd() @@ -46,6 +47,7 @@ def init(api=False): log_init() general_vars_init(api) + def conf_file_init(api=False): global CONF_FILE if api: @@ -66,24 +68,24 @@ def general_vars_init(api=False): # Check Section in Configuration File InputValidation.validate_configuration_file_section( - files.GENERAL, - "Section " + files.GENERAL + - "is not present in configuration file") + files.GENERAL, + "Section " + files.GENERAL + + "is not present in configuration file") InputValidation.validate_configuration_file_section( - files.OPENSTACK, - "Section " + files.OPENSTACK + - "is not present in configuration file") + files.OPENSTACK, + "Section " + files.OPENSTACK + + "is not present in configuration file") TEMPLATE_DIR = '/tmp/heat_templates/' if not api: # Validate template name InputValidation.validate_configuration_file_parameter( - files.GENERAL, - files.TEMPLATE_NAME, - "Parameter " + files.TEMPLATE_NAME + - "is not present in configuration file") + files.GENERAL, + files.TEMPLATE_NAME, + "Parameter " + files.TEMPLATE_NAME + + "is not present in configuration file") TEMPLATE_NAME = CONF_FILE.get_variable(files.GENERAL, files.TEMPLATE_NAME) InputValidation.validate_file_exist( @@ -112,6 +114,7 @@ def log_init(): # Configuration file access # ------------------------------------------------------ + class ConfigurationFile: """ Used to extract data from the configuration file @@ -188,6 +191,7 @@ class ConfigurationFile: # Manage files # ------------------------------------------------------ + def get_heat_template_params(): """ Returns the list of deployment parameters from the configuration file @@ -203,6 +207,7 @@ def get_heat_template_params(): files.DEPLOYMENT_PARAMETERS, param) return testcase_parameters + def get_testcase_params(): """ Returns the list of testcase parameters from the configuration file @@ -216,6 +221,7 @@ def get_testcase_params(): files.TESTCASE_PARAMETERS, param) return testcase_parameters + def get_file_first_line(file_name): """ Returns the first line of a file @@ -254,6 +260,8 @@ def replace_in_file(file, text_to_search, text_to_replace): # ------------------------------------------------------ # Shell interaction # ------------------------------------------------------ + + def run_command(command): LOG.info("Running command: {}".format(command)) return os.system(command) @@ -262,15 +270,19 @@ def run_command(command): # Expose variables to other modules # ------------------------------------------------------ + def get_base_dir(): return BASE_DIR + def get_template_dir(): return TEMPLATE_DIR # ------------------------------------------------------ # Configuration Variables from Config File # ------------------------------------------------------ + + def get_deployment_configuration_variables_from_conf_file(): variables = dict() types = dict() @@ -289,13 +301,17 @@ def get_deployment_configuration_variables_from_conf_file(): # ------------------------------------------------------ # benchmarks from Config File # ------------------------------------------------------ + + def get_benchmarks_from_conf_file(): requested_benchmarks = list() - benchmarks = CONF_FILE.get_variable(files.GENERAL, files.BENCHMARKS).split(', ') + benchmarks = CONF_FILE.get_variable( + files.GENERAL, files.BENCHMARKS).split(', ') for benchmark in benchmarks: requested_benchmarks.append(benchmark) return requested_benchmarks + class InputValidation(object): @staticmethod diff --git a/utils/infra_setup/heat/consts/files.py b/utils/infra_setup/heat/consts/files.py index 2856650f..f148f103 100755 --- a/utils/infra_setup/heat/consts/files.py +++ b/utils/infra_setup/heat/consts/files.py @@ -12,6 +12,7 @@ # ------------------------------------------------------ GENERAL = 'General' + def get_sections(): return [ GENERAL, diff --git a/utils/infra_setup/heat/manager.py b/utils/infra_setup/heat/manager.py index 5902e8c4..f5a9b88d 100755 --- a/utils/infra_setup/heat/manager.py +++ b/utils/infra_setup/heat/manager.py @@ -13,6 +13,7 @@ from heatclient.common import template_utils import heat.common as common + class HeatManager: def __init__(self, credentials): @@ -26,14 +27,14 @@ class HeatManager: def heat_init(self): keystone = keystone_client.Client(username=self.user, - password=self.password, - tenant_name=self.project_id, - auth_url=self.auth_uri) + password=self.password, + tenant_name=self.project_id, + auth_url=self.auth_uri) auth_token = keystone.auth_token self.heat_url = keystone.service_catalog.url_for( service_type='orchestration') self.heat = heat_client.Client('1', endpoint=self.heat_url, - token=auth_token) + token=auth_token) def stacks_list(self, name=None): for stack in self.heat.stacks.list(): @@ -44,7 +45,8 @@ class HeatManager: def stack_generate(self, template_file, stack_name, parameters): self.heat_init() self.stacks_list() - tpl_files, template = template_utils.get_template_contents(template_file) + tpl_files, template = template_utils.get_template_contents( + template_file) fields = { 'template': template, diff --git a/utils/infra_setup/heat/template.py b/utils/infra_setup/heat/template.py index f05831de..f71e9166 100755 --- a/utils/infra_setup/heat/template.py +++ b/utils/infra_setup/heat/template.py @@ -15,6 +15,7 @@ import shutil import common import consts.parameters as parameters + class TreeNode: def __init__(self): @@ -80,6 +81,7 @@ class TreeNode: template_name = parameters.TEST_TEMPLATE_NAME + def generates_templates(base_heat_template, deployment_configuration): # parameters loaded from file template_dir = common.get_template_dir() @@ -148,7 +150,8 @@ def get_all_heat_templates(template_dir, template_extension): template_files = list() for dirname, dirnames, filenames in os.walk(template_dir): for filename in filenames: - if template_extension in filename and filename.endswith(template_extension) and template_name in filename: + if template_extension in filename and filename.endswith( + template_extension) and template_name in filename: template_files.append(filename) template_files.sort() return template_files diff --git a/utils/infra_setup/heat/tests/generate_template_test.py b/utils/infra_setup/heat/tests/generate_template_test.py index d4e0a234..83c905ad 100755 --- a/utils/infra_setup/heat/tests/generate_template_test.py +++ b/utils/infra_setup/heat/tests/generate_template_test.py @@ -16,6 +16,7 @@ sys.path.append("..") import template import common + def reset_common(): common.LOG = None common.CONF_FILE = None @@ -26,7 +27,9 @@ def reset_common(): common.TEMPLATE_NAME = None common.TEMPLATE_EXTENSION = None + class TestGeneratesTemplate(unittest.TestCase): + def setUp(self): self.deployment_configuration = { 'flavor': ['medium'] @@ -42,7 +45,8 @@ class TestGeneratesTemplate(unittest.TestCase): def test_generates_template_for_success(self, mock_template_dir, mock_log): tmp_generated_templates_dir = '/data/generated_templates/' - generated_templates_dir = "{}{}".format(os.getcwd(), tmp_generated_templates_dir) + generated_templates_dir = "{}{}".format( + os.getcwd(), tmp_generated_templates_dir) mock_template_dir.return_value = generated_templates_dir tmp_test_templates = '/data/test_templates/' test_templates = "{}{}".format(os.getcwd(), tmp_test_templates) @@ -69,7 +73,8 @@ class TestGeneratesTemplate(unittest.TestCase): @mock.patch('common.get_template_dir') def test_get_all_heat_templates_for_success(self, template_dir): tmp_generated_templates = '/data/generated_templates/' - generated_templates = "{}{}".format(os.getcwd(), tmp_generated_templates) + generated_templates = "{}{}".format( + os.getcwd(), tmp_generated_templates) template_dir.return_value = generated_templates extension = '.yaml' expected = ['test_template_1.yaml'] |