summaryrefslogtreecommitdiffstats
path: root/utils
diff options
context:
space:
mode:
Diffstat (limited to 'utils')
-rw-r--r--utils/dashboard/process_data.py41
-rwxr-xr-xutils/dashboard/rubbos_collector.py22
-rwxr-xr-xutils/dashboard/uploader.py8
-rw-r--r--utils/dispatcher/func.py2
-rwxr-xr-xutils/infra_setup/heat/common.py38
-rwxr-xr-xutils/infra_setup/heat/consts/files.py1
-rwxr-xr-xutils/infra_setup/heat/manager.py12
-rwxr-xr-xutils/infra_setup/heat/template.py5
-rwxr-xr-xutils/infra_setup/heat/tests/generate_template_test.py9
9 files changed, 88 insertions, 50 deletions
diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py
index d4b2ccfb..e71a1e82 100644
--- a/utils/dashboard/process_data.py
+++ b/utils/dashboard/process_data.py
@@ -13,6 +13,7 @@ import sys
from rubbos_collector import RubbosCollector
from uploader import Uploader
+
def printUsage():
print "Usage: python process_data.py required_params(**) optional_params([])"
print " ** -i|--input input_data_dir"
@@ -21,12 +22,14 @@ def printUsage():
print " [] -o|--output output_file"
print " [] -u|--upload yes|no"
+
def process(input_dir, suite_name):
result = dict()
if suite_name == "rubbos":
result = RubbosCollector().collect_data(input_dir)
return result
+
def writeResult(output_file, result):
f = open(output_file, "w")
if isinstance(result, list):
@@ -34,41 +37,43 @@ def writeResult(output_file, result):
f.write(str(elem) + "\n")
f.close()
+
def uploadResult(conf, suite_name, result):
Uploader(conf).upload_result(suite_name, result)
+
def main():
if len(sys.argv) < 7 or len(sys.argv) % 2 == 0:
printUsage()
- exit (1)
+ exit(1)
i = 1
params = dict()
while (i < len(sys.argv)):
- if sys.argv[i]=="-i" or sys.argv[i]=="--input":
- params["input"] = sys.argv[i+1]
- if sys.argv[i]=="-s" or sys.argv[i]=="--suite":
- params["suite"] = sys.argv[i+1]
- if sys.argv[i]=="-c" or sys.argv[i]=="--conf":
- params["conf"] = sys.argv[i+1]
- if sys.argv[i]=="-o" or sys.argv[i]=="--output":
- params["output"] = sys.argv[i+1]
- if sys.argv[i]=="-u" or sys.argv[i]=="--upload":
- params["upload"] = sys.argv[i+1]
- i = i+2
- if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")):
+ if sys.argv[i] == "-i" or sys.argv[i] == "--input":
+ params["input"] = sys.argv[i + 1]
+ if sys.argv[i] == "-s" or sys.argv[i] == "--suite":
+ params["suite"] = sys.argv[i + 1]
+ if sys.argv[i] == "-c" or sys.argv[i] == "--conf":
+ params["conf"] = sys.argv[i + 1]
+ if sys.argv[i] == "-o" or sys.argv[i] == "--output":
+ params["output"] = sys.argv[i + 1]
+ if sys.argv[i] == "-u" or sys.argv[i] == "--upload":
+ params["upload"] = sys.argv[i + 1]
+ i = i + 2
+ if not("input" in params and "suite" in params and "conf" in params):
print "Lack some required parameters."
- exit (1)
+ exit(1)
result = process(params["input"], params["suite"])
print "Results:"
for elem in result:
print elem
- if params.has_key("output"):
- writeResult(params["output"],result)
+ if "output" in params:
+ writeResult(params["output"], result)
- if params.has_key("upload") and params["upload"].lower()=="yes":
+ if "upload" in params and params["upload"].lower() == "yes":
uploadResult(params["conf"], params["suite"], result)
-if __name__=="__main__":
+if __name__ == "__main__":
main()
diff --git a/utils/dashboard/rubbos_collector.py b/utils/dashboard/rubbos_collector.py
index c9851739..d9f86032 100755
--- a/utils/dashboard/rubbos_collector.py
+++ b/utils/dashboard/rubbos_collector.py
@@ -10,16 +10,18 @@
import subprocess as subp
+
def exec_shell(cmd):
- out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate()
+ out, err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate()
return out.strip()
+
def get_onetime_data(dir_name):
cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
client_node_num = int(exec_shell(cmd))
cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
each_client_num = int(exec_shell(cmd))
- total_client = (client_node_num+1) * each_client_num
+ total_client = (client_node_num + 1) * each_client_num
cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
throughput = int(exec_shell(cmd))
@@ -39,11 +41,15 @@ class RubbosCollector(object):
pass
def collect_data(self, data_home):
- cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home
+ cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home
result = []
for subdir in exec_shell(cmd).split('\n'):
- total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir)
- result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request})
- result.sort(key=lambda x:x['client'])
-
- return result;
+ total_client, throughput, request, error_request = get_onetime_data(
+ data_home + '/' + subdir)
+ result.append({'client': total_client,
+ 'throughput': throughput,
+ 'request': request,
+ 'error_request': error_request})
+ result.sort(key=lambda x: x['client'])
+
+ return result
diff --git a/utils/dashboard/uploader.py b/utils/dashboard/uploader.py
index 07862fed..a4686560 100755
--- a/utils/dashboard/uploader.py
+++ b/utils/dashboard/uploader.py
@@ -29,7 +29,6 @@ class Uploader(object):
self.result['version'] = dashboard_conf['version']
self.target = dashboard_conf['target']
-
def upload_result(self, case_name, raw_data):
if self.target == '':
print('No target was set, so no data will be posted.')
@@ -43,7 +42,9 @@ class Uploader(object):
data=json.dumps(self.result),
headers=self.headers,
timeout=self.timeout)
- print('Test result posting finished with status code %d.' % res.status_code)
+ print(
+ 'Test result posting finished with status code %d.' %
+ res.status_code)
except Exception as err:
print ('Failed to record result data: %s', err)
@@ -55,10 +56,9 @@ def _test():
print ("no argumens input!!")
exit(1)
- with open(sys.argv[1],'r') as stream:
+ with open(sys.argv[1], 'r') as stream:
data = json.load(stream)
Uploader().upload_result(data)
if __name__ == "__main__":
_test()
-
diff --git a/utils/dispatcher/func.py b/utils/dispatcher/func.py
index 71830bf2..cb907e5e 100644
--- a/utils/dispatcher/func.py
+++ b/utils/dispatcher/func.py
@@ -1,4 +1,4 @@
-#Copyright 2013: Mirantis Inc.
+# Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
diff --git a/utils/infra_setup/heat/common.py b/utils/infra_setup/heat/common.py
index 24de893f..c4a78249 100755
--- a/utils/infra_setup/heat/common.py
+++ b/utils/infra_setup/heat/common.py
@@ -34,6 +34,7 @@ TEMPLATE_EXTENSION = None
# Initialization and Input 'heat_templates/'validation
# ------------------------------------------------------
+
def init(api=False):
global BASE_DIR
# BASE_DIR = os.getcwd()
@@ -46,6 +47,7 @@ def init(api=False):
log_init()
general_vars_init(api)
+
def conf_file_init(api=False):
global CONF_FILE
if api:
@@ -66,24 +68,24 @@ def general_vars_init(api=False):
# Check Section in Configuration File
InputValidation.validate_configuration_file_section(
- files.GENERAL,
- "Section " + files.GENERAL +
- "is not present in configuration file")
+ files.GENERAL,
+ "Section " + files.GENERAL +
+ "is not present in configuration file")
InputValidation.validate_configuration_file_section(
- files.OPENSTACK,
- "Section " + files.OPENSTACK +
- "is not present in configuration file")
+ files.OPENSTACK,
+ "Section " + files.OPENSTACK +
+ "is not present in configuration file")
TEMPLATE_DIR = '/tmp/heat_templates/'
if not api:
# Validate template name
InputValidation.validate_configuration_file_parameter(
- files.GENERAL,
- files.TEMPLATE_NAME,
- "Parameter " + files.TEMPLATE_NAME +
- "is not present in configuration file")
+ files.GENERAL,
+ files.TEMPLATE_NAME,
+ "Parameter " + files.TEMPLATE_NAME +
+ "is not present in configuration file")
TEMPLATE_NAME = CONF_FILE.get_variable(files.GENERAL,
files.TEMPLATE_NAME)
InputValidation.validate_file_exist(
@@ -112,6 +114,7 @@ def log_init():
# Configuration file access
# ------------------------------------------------------
+
class ConfigurationFile:
"""
Used to extract data from the configuration file
@@ -188,6 +191,7 @@ class ConfigurationFile:
# Manage files
# ------------------------------------------------------
+
def get_heat_template_params():
"""
Returns the list of deployment parameters from the configuration file
@@ -203,6 +207,7 @@ def get_heat_template_params():
files.DEPLOYMENT_PARAMETERS, param)
return testcase_parameters
+
def get_testcase_params():
"""
Returns the list of testcase parameters from the configuration file
@@ -216,6 +221,7 @@ def get_testcase_params():
files.TESTCASE_PARAMETERS, param)
return testcase_parameters
+
def get_file_first_line(file_name):
"""
Returns the first line of a file
@@ -254,6 +260,8 @@ def replace_in_file(file, text_to_search, text_to_replace):
# ------------------------------------------------------
# Shell interaction
# ------------------------------------------------------
+
+
def run_command(command):
LOG.info("Running command: {}".format(command))
return os.system(command)
@@ -262,15 +270,19 @@ def run_command(command):
# Expose variables to other modules
# ------------------------------------------------------
+
def get_base_dir():
return BASE_DIR
+
def get_template_dir():
return TEMPLATE_DIR
# ------------------------------------------------------
# Configuration Variables from Config File
# ------------------------------------------------------
+
+
def get_deployment_configuration_variables_from_conf_file():
variables = dict()
types = dict()
@@ -289,13 +301,17 @@ def get_deployment_configuration_variables_from_conf_file():
# ------------------------------------------------------
# benchmarks from Config File
# ------------------------------------------------------
+
+
def get_benchmarks_from_conf_file():
requested_benchmarks = list()
- benchmarks = CONF_FILE.get_variable(files.GENERAL, files.BENCHMARKS).split(', ')
+ benchmarks = CONF_FILE.get_variable(
+ files.GENERAL, files.BENCHMARKS).split(', ')
for benchmark in benchmarks:
requested_benchmarks.append(benchmark)
return requested_benchmarks
+
class InputValidation(object):
@staticmethod
diff --git a/utils/infra_setup/heat/consts/files.py b/utils/infra_setup/heat/consts/files.py
index 2856650f..f148f103 100755
--- a/utils/infra_setup/heat/consts/files.py
+++ b/utils/infra_setup/heat/consts/files.py
@@ -12,6 +12,7 @@
# ------------------------------------------------------
GENERAL = 'General'
+
def get_sections():
return [
GENERAL,
diff --git a/utils/infra_setup/heat/manager.py b/utils/infra_setup/heat/manager.py
index 5902e8c4..f5a9b88d 100755
--- a/utils/infra_setup/heat/manager.py
+++ b/utils/infra_setup/heat/manager.py
@@ -13,6 +13,7 @@ from heatclient.common import template_utils
import heat.common as common
+
class HeatManager:
def __init__(self, credentials):
@@ -26,14 +27,14 @@ class HeatManager:
def heat_init(self):
keystone = keystone_client.Client(username=self.user,
- password=self.password,
- tenant_name=self.project_id,
- auth_url=self.auth_uri)
+ password=self.password,
+ tenant_name=self.project_id,
+ auth_url=self.auth_uri)
auth_token = keystone.auth_token
self.heat_url = keystone.service_catalog.url_for(
service_type='orchestration')
self.heat = heat_client.Client('1', endpoint=self.heat_url,
- token=auth_token)
+ token=auth_token)
def stacks_list(self, name=None):
for stack in self.heat.stacks.list():
@@ -44,7 +45,8 @@ class HeatManager:
def stack_generate(self, template_file, stack_name, parameters):
self.heat_init()
self.stacks_list()
- tpl_files, template = template_utils.get_template_contents(template_file)
+ tpl_files, template = template_utils.get_template_contents(
+ template_file)
fields = {
'template': template,
diff --git a/utils/infra_setup/heat/template.py b/utils/infra_setup/heat/template.py
index f05831de..f71e9166 100755
--- a/utils/infra_setup/heat/template.py
+++ b/utils/infra_setup/heat/template.py
@@ -15,6 +15,7 @@ import shutil
import common
import consts.parameters as parameters
+
class TreeNode:
def __init__(self):
@@ -80,6 +81,7 @@ class TreeNode:
template_name = parameters.TEST_TEMPLATE_NAME
+
def generates_templates(base_heat_template, deployment_configuration):
# parameters loaded from file
template_dir = common.get_template_dir()
@@ -148,7 +150,8 @@ def get_all_heat_templates(template_dir, template_extension):
template_files = list()
for dirname, dirnames, filenames in os.walk(template_dir):
for filename in filenames:
- if template_extension in filename and filename.endswith(template_extension) and template_name in filename:
+ if template_extension in filename and filename.endswith(
+ template_extension) and template_name in filename:
template_files.append(filename)
template_files.sort()
return template_files
diff --git a/utils/infra_setup/heat/tests/generate_template_test.py b/utils/infra_setup/heat/tests/generate_template_test.py
index d4e0a234..83c905ad 100755
--- a/utils/infra_setup/heat/tests/generate_template_test.py
+++ b/utils/infra_setup/heat/tests/generate_template_test.py
@@ -16,6 +16,7 @@ sys.path.append("..")
import template
import common
+
def reset_common():
common.LOG = None
common.CONF_FILE = None
@@ -26,7 +27,9 @@ def reset_common():
common.TEMPLATE_NAME = None
common.TEMPLATE_EXTENSION = None
+
class TestGeneratesTemplate(unittest.TestCase):
+
def setUp(self):
self.deployment_configuration = {
'flavor': ['medium']
@@ -42,7 +45,8 @@ class TestGeneratesTemplate(unittest.TestCase):
def test_generates_template_for_success(self, mock_template_dir,
mock_log):
tmp_generated_templates_dir = '/data/generated_templates/'
- generated_templates_dir = "{}{}".format(os.getcwd(), tmp_generated_templates_dir)
+ generated_templates_dir = "{}{}".format(
+ os.getcwd(), tmp_generated_templates_dir)
mock_template_dir.return_value = generated_templates_dir
tmp_test_templates = '/data/test_templates/'
test_templates = "{}{}".format(os.getcwd(), tmp_test_templates)
@@ -69,7 +73,8 @@ class TestGeneratesTemplate(unittest.TestCase):
@mock.patch('common.get_template_dir')
def test_get_all_heat_templates_for_success(self, template_dir):
tmp_generated_templates = '/data/generated_templates/'
- generated_templates = "{}{}".format(os.getcwd(), tmp_generated_templates)
+ generated_templates = "{}{}".format(
+ os.getcwd(), tmp_generated_templates)
template_dir.return_value = generated_templates
extension = '.yaml'
expected = ['test_template_1.yaml']