diff options
Diffstat (limited to 'yardstick/vTC/apexlake/experimental_framework/common.py')
| -rw-r--r-- | yardstick/vTC/apexlake/experimental_framework/common.py | 570 |
1 files changed, 570 insertions, 0 deletions
diff --git a/yardstick/vTC/apexlake/experimental_framework/common.py b/yardstick/vTC/apexlake/experimental_framework/common.py new file mode 100644 index 000000000..97f5bee93 --- /dev/null +++ b/yardstick/vTC/apexlake/experimental_framework/common.py @@ -0,0 +1,570 @@ +# Copyright (c) 2015 Intel Research and Development Ireland Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import ConfigParser +import logging +import fileinput +from experimental_framework.constants import conf_file_sections as cf +from experimental_framework.constants import framework_parameters as fp + + +# ------------------------------------------------------ +# List of common variables +# ------------------------------------------------------ + +LOG = None +CONF_FILE = None +DEPLOYMENT_UNIT = None +ITERATIONS = None + +BASE_DIR = None +RESULT_DIR = None +TEMPLATE_DIR = None +TEMPLATE_NAME = None +TEMPLATE_FILE_EXTENSION = None + +PKTGEN = None +PKTGEN_DIR = None +PKTGEN_DPDK_DIRECTORY = None +PKTGEN_PROGRAM = None +PKTGEN_COREMASK = None +PKTGEN_MEMCHANNEL = None +PKTGEN_BUS_SLOT_NIC_1 = None +PKTGEN_BUS_SLOT_NIC_2 = None + +INFLUXDB_IP = None +INFLUXDB_PORT = None +INFLUXDB_DB_NAME = None + + +# ------------------------------------------------------ +# Initialization and Input 'heat_templates/'validation +# ------------------------------------------------------ + +def init(api=False): + global BASE_DIR + BASE_DIR = os.getcwd() + BASE_DIR = BASE_DIR.replace('/experimental_framework', '') + BASE_DIR = InputValidation.validate_directory_exist_and_format( + BASE_DIR, "Error 000001") + + init_conf_file(api) + init_general_vars() + init_log() + if len(CONF_FILE.get_variable_list(cf.CFS_PKTGEN)) > 0: + init_pktgen() + + +def init_conf_file(api=False): + global CONF_FILE + if api: + CONF_FILE = ConfigurationFile(cf.get_sections_api()) + else: + CONF_FILE = ConfigurationFile(cf.get_sections()) + + +def init_general_vars(): + global TEMPLATE_FILE_EXTENSION + global TEMPLATE_NAME + global TEMPLATE_DIR + global RESULT_DIR + global ITERATIONS + + TEMPLATE_FILE_EXTENSION = '.yaml' + + # Check Section in Configuration File + InputValidation.\ + validate_configuration_file_section( + cf.CFS_GENERAL, + "Section " + cf.CFS_GENERAL + + "is not present in configuration file") + + TEMPLATE_DIR = BASE_DIR + 'heat_templates/' + + # Validate template name + InputValidation.\ + validate_configuration_file_parameter( + cf.CFS_GENERAL, + cf.CFSG_TEMPLATE_NAME, + "Parameter " + cf.CFSG_TEMPLATE_NAME + + "is not present in configuration file") + + TEMPLATE_NAME = CONF_FILE.get_variable(cf.CFS_GENERAL, + cf.CFSG_TEMPLATE_NAME) + InputValidation.validate_file_exist( + TEMPLATE_DIR + TEMPLATE_NAME, + "The provided template file does not exist") + RESULT_DIR = BASE_DIR + 'results/' + + # Validate and assign Iterations + if cf.CFSG_ITERATIONS in CONF_FILE.get_variable_list(cf.CFS_GENERAL): + ITERATIONS = int(CONF_FILE.get_variable(cf.CFS_GENERAL, + cf.CFSG_ITERATIONS)) + else: + ITERATIONS = 1 + + +def init_log(): + global LOG + if cf.CFSG_DEBUG in CONF_FILE.get_variable_list(cf.CFS_GENERAL) and \ + CONF_FILE.get_variable(cf.CFS_GENERAL, cf.CFSG_DEBUG): + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + LOG = logging.getLogger() + log_formatter = logging.Formatter("%(asctime)s --- %(message)s") + file_handler = logging.FileHandler("{0}/{1}.log".format("./", "benchmark")) + file_handler.setFormatter(log_formatter) + file_handler.setLevel(logging.DEBUG) + LOG.addHandler(file_handler) + + +# ------------------------------------------------------ +# InfluxDB conf variables +# ------------------------------------------------------ +def init_influxdb(): + global INFLUXDB_IP + global INFLUXDB_PORT + global INFLUXDB_DB_NAME + + INFLUXDB_IP = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_IP) + INFLUXDB_PORT = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_PORT) + INFLUXDB_DB_NAME = CONF_FILE.get_variable(cf.CFS_INFLUXDB, + cf.CFSI_IDB_DB_NAME) + + +# ------------------------------------------------------ +# Packet Generator conf variables +# ------------------------------------------------------ +def init_pktgen(): + global PKTGEN + global PKTGEN_DIR + global PKTGEN_PROGRAM + global PKTGEN_COREMASK + global PKTGEN_MEMCHANNEL + global PKTGEN_BUS_SLOT_NIC_1 + global PKTGEN_BUS_SLOT_NIC_2 + global PKTGEN_DPDK_DIRECTORY + + msg = "Section {} is not present in the configuration file".\ + format(cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_section(cf.CFS_PKTGEN, msg) + + pktgen_var_list = CONF_FILE.get_variable_list(cf.CFS_PKTGEN) + PKTGEN = 'dpdk_pktgen' # default value + if cf.CFSP_PACKET_GENERATOR in pktgen_var_list: + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_PACKET_GENERATOR, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR, msg) + PKTGEN = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR) + + if PKTGEN not in fp.get_supported_packet_generators(): + raise ValueError('The specified packet generator is not supported ' + 'by the framework') + + # Check if the packet gen is dpdk_pktgen + if PKTGEN == cf.CFSP_PG_DPDK: + # Validation of DPDK pktgen directory + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_PKTGEN_DIRECTORY, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY, msg) + PKTGEN_DIR = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY) + msg = "The directory {} does not exist.".format(PKTGEN_DIR) + PKTGEN_DIR = InputValidation.validate_directory_exist_and_format( + PKTGEN_DIR, msg) + + # Validation of the DPDK program name + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_PROGRAM_NAME, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME, msg) + PKTGEN_PROGRAM = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME) + + # Validation of the DPDK Coremask parameter + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_COREMASK, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK, msg) + PKTGEN_COREMASK = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK) + + # Validation of the DPDK Memory Channel parameter + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_MEMORY_CHANNEL, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL, msg) + PKTGEN_MEMCHANNEL = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL) + + # Validation of the DPDK Bus Slot 1 + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_BUS_SLOT_NIC_1, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1, msg) + PKTGEN_BUS_SLOT_NIC_1 = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1) + + # Validation of the DPDK Bus Slot 2 + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_BUS_SLOT_NIC_2, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2, msg) + PKTGEN_BUS_SLOT_NIC_2 = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2) + + # Validation of DPDK directory parameter + msg = "Parameter {} is not present in section {}".format( + cf.CFSP_DPDK_DPDK_DIRECTORY, cf.CFS_PKTGEN) + InputValidation.validate_configuration_file_parameter( + cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY, msg) + PKTGEN_DPDK_DIRECTORY = CONF_FILE.get_variable( + cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY) + msg = "Directory {} does not exist".format( + cf.CFSP_DPDK_DPDK_DIRECTORY) + PKTGEN_DPDK_DIRECTORY = InputValidation.\ + validate_directory_exist_and_format(PKTGEN_DPDK_DIRECTORY, msg) + + +# ------------------------------------------------------ +# Configuration file access +# ------------------------------------------------------ + +class ConfigurationFile: + """ + Used to extract data from the configuration file + """ + + def __init__(self, sections, config_file='conf.cfg'): + """ + Reads configuration file sections + + :param sections: list of strings representing the sections to be + loaded + :param config_file: name of the configuration file (string) + :return: None + """ + InputValidation.validate_string( + config_file, "The configuration file name must be a string") + config_file = BASE_DIR + config_file + InputValidation.validate_file_exist( + config_file, 'The provided configuration file does not exist') + self.config = ConfigParser.ConfigParser() + self.config.read(config_file) + for section in sections: + setattr( + self, section, ConfigurationFile. + _config_section_map(section, self.config)) + + @staticmethod + def _config_section_map(section, config_file): + """ + Returns a dictionary with the configuration values for the specific + section + + :param section: section to be loaded (string) + :param config_file: name of the configuration file (string) + :return: dict + """ + dict1 = dict() + options = config_file.options(section) + for option in options: + dict1[option] = config_file.get(section, option) + return dict1 + + def get_variable(self, section, variable_name): + """ + Returns the value correspondent to a variable + + :param section: section to be loaded (string) + :param variable_name: name of the variable (string) + :return: string + """ + message = "The variable name must be a string" + InputValidation.validate_string(variable_name, message) + if variable_name in self.get_variable_list(section): + sect = getattr(self, section) + return sect[variable_name] + else: + exc_msg = 'Parameter {} is not in the {} section of the conf file' + exc_msg.format(variable_name, section) + raise ValueError(exc_msg) + + def get_variable_list(self, section): + """ + Returns the list of the available variables in a section + :param section: section to be loaded (string) + :return: list + """ + try: + return getattr(self, section) + except: + msg = 'Section {} not found in the configuration file'.\ + format(section) + raise ValueError(msg) + + +# ------------------------------------------------------ +# Get OpenStack Credentials +# ------------------------------------------------------ +def get_credentials(): + """ + Returns the credentials for OpenStack access from the configuration file + :return: dictionary + """ + credentials = dict() + credentials[cf.CFSO_IP_CONTROLLER] = CONF_FILE.get_variable( + cf.CFS_OPENSTACK, cf.CFSO_IP_CONTROLLER) + credentials[cf.CFSO_HEAT_URL] = CONF_FILE.get_variable( + cf.CFS_OPENSTACK, cf.CFSO_HEAT_URL) + credentials[cf.CFSO_USER] = CONF_FILE.get_variable( + cf.CFS_OPENSTACK, cf.CFSO_USER) + credentials[cf.CFSO_PASSWORD] = CONF_FILE.get_variable( + cf.CFS_OPENSTACK, cf.CFSO_PASSWORD) + credentials[cf.CFSO_AUTH_URI] = CONF_FILE.get_variable( + cf.CFS_OPENSTACK, cf.CFSO_AUTH_URI) + credentials[cf.CFSO_PROJECT] = CONF_FILE.get_variable( + cf.CFS_OPENSTACK, cf.CFSO_PROJECT) + return credentials + + +# ------------------------------------------------------ +# Manage files +# ------------------------------------------------------ + +def get_heat_template_params(): + """ + Returns the list of deployment parameters from the configuration file + for the heat template + + :return: dict + """ + heat_parameters_list = CONF_FILE.get_variable_list( + cf.CFS_DEPLOYMENT_PARAMETERS) + testcase_parameters = dict() + for param in heat_parameters_list: + testcase_parameters[param] = CONF_FILE.get_variable( + cf.CFS_DEPLOYMENT_PARAMETERS, param) + return testcase_parameters + + +def get_testcase_params(): + """ + Returns the list of testcase parameters from the configuration file + + :return: dict + """ + testcase_parameters = dict() + parameters = CONF_FILE.get_variable_list(cf.CFS_TESTCASE_PARAMETERS) + for param in parameters: + testcase_parameters[param] = CONF_FILE.get_variable( + cf.CFS_TESTCASE_PARAMETERS, param) + return testcase_parameters + + +def get_file_first_line(file_name): + """ + Returns the first line of a file + + :param file_name: name of the file to be read (str) + :return: str + """ + message = "The name of the file must be a string" + InputValidation.validate_string(file_name, message) + message = 'The file {} does not exist'.format(file_name) + InputValidation.validate_file_exist(file_name, message) + res = open(file_name, 'r') + return res.readline() + + +def replace_in_file(file, text_to_search, text_to_replace): + """ + Replaces a string within a file + + :param file: name of the file (str) + :param text_to_search: text to be replaced + :param text_to_replace: new text that will replace the previous + :return: None + """ + message = 'The text to be replaced in the file must be a string' + InputValidation.validate_string(text_to_search, message) + message = 'The text to replace in the file must be a string' + InputValidation.validate_string(text_to_replace, message) + message = "The name of the file must be a string" + InputValidation.validate_string(file, message) + message = "The file does not exist" + InputValidation.validate_file_exist(file, message) + for line in fileinput.input(file, inplace=True): + print(line.replace(text_to_search, text_to_replace).rstrip()) + + +# ------------------------------------------------------ +# Shell interaction +# ------------------------------------------------------ +def run_command(command): + LOG.info("Running command: " + command) + return os.system(command) + + +def push_data_influxdb(data): + ip = INFLUXDB_IP + port = INFLUXDB_PORT + db_name = INFLUXDB_DB_NAME + command = "curl -i -XPOST 'http://{}:{}/write?db={}' " \ + "--data-binary {}".format(ip, port, db_name, data) + run_command(command) + + +# ------------------------------------------------------ +# Expose variables to other modules +# ------------------------------------------------------ + +def get_base_dir(): + return BASE_DIR + + +def get_template_dir(): + return TEMPLATE_DIR + + +def get_dpdk_pktgen_vars(): + if not (PKTGEN == 'dpdk_pktgen'): + return dict() + ret_val = dict() + ret_val[cf.CFSP_DPDK_PKTGEN_DIRECTORY] = PKTGEN_DIR + ret_val[cf.CFSP_DPDK_PROGRAM_NAME] = PKTGEN_PROGRAM + ret_val[cf.CFSP_DPDK_COREMASK] = PKTGEN_COREMASK + ret_val[cf.CFSP_DPDK_MEMORY_CHANNEL] = PKTGEN_MEMCHANNEL + ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_1] = PKTGEN_BUS_SLOT_NIC_1 + ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_2] = PKTGEN_BUS_SLOT_NIC_2 + ret_val[cf.CFSP_DPDK_DPDK_DIRECTORY] = PKTGEN_DPDK_DIRECTORY + return ret_val + + +# ------------------------------------------------------ +# Configuration Variables from Config File +# ------------------------------------------------------ +def get_deployment_configuration_variables_from_conf_file(): + variables = dict() + types = dict() + all_variables = CONF_FILE.get_variable_list(cf.CFS_EXPERIMENT_VNF) + for var in all_variables: + v = CONF_FILE.get_variable(cf.CFS_EXPERIMENT_VNF, var) + type = re.findall(r'@\w*', v) + values = re.findall(r'\"(.+?)\"', v) + variables[var] = values + try: + types[var] = type[0][1:] + except IndexError: + LOG.debug("No type has been specified for variable " + var) + return variables + + +# ------------------------------------------------------ +# benchmarks from Config File +# ------------------------------------------------------ +def get_benchmarks_from_conf_file(): + requested_benchmarks = list() + benchmarks = \ + CONF_FILE.get_variable(cf.CFS_GENERAL, cf.CFSG_BENCHMARKS).split(', ') + for benchmark in benchmarks: + requested_benchmarks.append(benchmark) + return requested_benchmarks + + +class InputValidation(object): + + @staticmethod + def validate_string(param, message): + if not isinstance(param, str): + raise ValueError(message) + return True + + @staticmethod + def validate_integer(param, message): + if not isinstance(param, int): + raise ValueError(message) + return True + + @staticmethod + def validate_dictionary(param, message): + if not isinstance(param, dict): + raise ValueError(message) + return True + + @staticmethod + def validate_file_exist(file_name, message): + if not os.path.isfile(file_name): + raise ValueError(message + ' ' + file_name) + return True + + @staticmethod + def validate_directory_exist_and_format(directory, message): + if not os.path.isdir(directory): + raise ValueError(message) + if not directory.endswith('/'): + return directory + '/' + return directory + + @staticmethod + def validate_configuration_file_parameter(section, parameter, message): + params = CONF_FILE.get_variable_list(section) + if parameter not in params: + raise ValueError(message) + return True + + @staticmethod + def validate_configuration_file_section(section, message): + if section not in cf.get_sections(): + raise ValueError(message) + return True + + @staticmethod + def validate_boolean(boolean, message): + if isinstance(boolean, bool): + return boolean + if isinstance(boolean, str): + if boolean == 'True': + return True + if boolean == 'False': + return False + raise ValueError(message) + + @staticmethod + def validate_os_credentials(credentials): + if not isinstance(credentials, dict): + raise ValueError( + 'The provided openstack_credentials ' + 'variable must be in dictionary format') + + credential_keys = ['ip_controller', 'heat_url', 'user', 'password', + 'auth_uri', 'project'] + missing = [ + credential_key + for credential_key in credential_keys + if credential_key not in credentials.keys() + ] + if len(missing) == 0: + return True + msg = 'OpenStack Credentials Error! ' \ + 'The following parameters are missing: {}'.\ + format(", ".join(missing)) + raise ValueError(msg) |