diff options
| -rw-r--r-- | functest/ci/config_functest.yaml | 2 | ||||
| -rw-r--r-- | functest/energy/energy.py | 31 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/blacklist.txt | 3 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 208 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/refstack_client/refstack_client.py | 129 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/vping/vping_ssh.py | 78 | ||||
| -rw-r--r-- | functest/opnfv_tests/vnf/ims/cloudify_ims.py | 77 | ||||
| -rw-r--r-- | functest/tests/unit/energy/test_functest_energy.py | 5 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/rally/test_rally.py | 45 | ||||
| -rw-r--r-- | functest/utils/openstack_utils.py | 4 |
10 files changed, 363 insertions, 219 deletions
diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml index 4833c5c8d..e26b31398 100644 --- a/functest/ci/config_functest.yaml +++ b/functest/ci/config_functest.yaml @@ -200,6 +200,6 @@ results: test_db_url: http://testresults.opnfv.org/test/api/v1/results energy_recorder: - api_url: http://opnfv.fr:8888/resources + api_url: http://energy.opnfv.fr/resources api_user: "" api_password: "" diff --git a/functest/energy/energy.py b/functest/energy/energy.py index 71b712393..372c1d32f 100644 --- a/functest/energy/energy.py +++ b/functest/energy/energy.py @@ -13,11 +13,24 @@ import json import logging import urllib + +from functools import wraps import requests import functest.utils.functest_utils as ft_utils +def finish_session(current_scenario): + """Finish a recording session.""" + if current_scenario is None: + EnergyRecorder.stop() + else: + EnergyRecorder.submit_scenario( + current_scenario["scenario"], + current_scenario["step"] + ) + + def enable_recording(method): """ Record energy during method execution. @@ -30,6 +43,7 @@ def enable_recording(method): .. note:: "method" should belong to a class having a "case_name" attribute """ + @wraps(method) def wrapper(*args): """ Record energy during method execution (implementation). @@ -38,14 +52,12 @@ def enable_recording(method): """ current_scenario = EnergyRecorder.get_current_scenario() EnergyRecorder.start(args[0].case_name) - return_value = method(*args) - if current_scenario is None: - EnergyRecorder.stop() - else: - EnergyRecorder.submit_scenario( - current_scenario["scenario"], - current_scenario["step"] - ) + try: + return_value = method(*args) + finish_session(current_scenario) + except Exception: # pylint: disable=broad-except + finish_session(current_scenario) + raise return return_value return wrapper @@ -246,7 +258,6 @@ class EnergyRecorder(object): """Get current running scenario (if any, None else).""" EnergyRecorder.logger.debug("Getting current scenario") return_value = None - print "In get current" try: # Ensure that connectyvity settings are loaded EnergyRecorder.load_config() @@ -263,13 +274,11 @@ class EnergyRecorder(object): log_msg = log_msg.format( EnergyRecorder.energy_recorder_api["uri"]) EnergyRecorder.logger.error(log_msg) - print log_msg return_value = None else: log_msg = "Error while getting current scenario\n{}" log_msg = log_msg.format(response.text) EnergyRecorder.logger.error(log_msg) - print log_msg return_value = None except Exception: # pylint: disable=broad-except # Default exception handler to ensure that method diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt index 3a17fa616..95bea2b7a 100644 --- a/functest/opnfv_tests/openstack/rally/blacklist.txt +++ b/functest/opnfv_tests/openstack/rally/blacklist.txt @@ -1,8 +1,7 @@ scenario: - scenarios: - - os-nosdn-lxd-ha - - os-nosdn-lxd-noha + - '^os-nosdn-lxd-(no)?ha$' installers: - joid tests: diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 24c9147c8..6b7c49ca7 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -8,27 +8,32 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +"""Rally testcases implementation.""" + from __future__ import division import json import logging import os -import pkg_resources import re import subprocess import time import iniparse +import pkg_resources import yaml from functest.core import testcase +from functest.energy import energy from functest.utils.constants import CONST import functest.utils.openstack_utils as os_utils -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class RallyBase(testcase.OSGCTestCase): + """Base class form Rally testcases implementation.""" + TESTS = ['authenticate', 'glance', 'cinder', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'vm', 'all'] GLANCE_IMAGE_NAME = CONST.__getattribute__('openstack_image_name') @@ -64,6 +69,7 @@ class RallyBase(testcase.OSGCTestCase): RALLY_ROUTER_NAME = CONST.__getattribute__('rally_router_name') def __init__(self, **kwargs): + """Initialize RallyBase object.""" super(RallyBase, self).__init__(**kwargs) self.mode = '' self.summary = [] @@ -74,6 +80,12 @@ class RallyBase(testcase.OSGCTestCase): self.network_dict = {} self.volume_type = None self.smoke = None + self.test_name = None + self.image_exists = None + self.image_id = None + self.start_time = None + self.result = None + self.details = None def _build_task_args(self, test_file_name): task_args = {'service_list': [test_file_name]} @@ -117,7 +129,7 @@ class RallyBase(testcase.OSGCTestCase): raise Exception("The scenario '%s' does not exist." % scenario_file_name) - logger.debug('Scenario fetched from : {}'.format(scenario_file_name)) + LOGGER.debug('Scenario fetched from : %s', scenario_file_name) test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name) if not os.path.exists(self.TEMP_DIR): @@ -129,7 +141,8 @@ class RallyBase(testcase.OSGCTestCase): @staticmethod def get_task_id(cmd_raw): """ - get task id from command rally result + Get task id from command rally result. + :param cmd_raw: :return: task_id as string """ @@ -144,7 +157,8 @@ class RallyBase(testcase.OSGCTestCase): @staticmethod def task_succeed(json_raw): """ - Parse JSON from rally JSON results + Parse JSON from rally JSON results. + :param json_raw: :return: Bool """ @@ -161,6 +175,7 @@ class RallyBase(testcase.OSGCTestCase): @staticmethod def live_migration_supported(): + """Determine is live migration is supported.""" config = iniparse.ConfigParser() if (config.read(RallyBase.TEMPEST_CONF_FILE) and config.has_section('compute-feature-enabled') and @@ -173,6 +188,7 @@ class RallyBase(testcase.OSGCTestCase): @staticmethod def get_cmd_output(proc): + """Get command stdout.""" result = "" while proc.poll() is None: line = proc.stdout.readline() @@ -181,6 +197,7 @@ class RallyBase(testcase.OSGCTestCase): @staticmethod def excl_scenario(): + """Exclude scenario.""" black_tests = [] try: with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file: @@ -188,22 +205,45 @@ class RallyBase(testcase.OSGCTestCase): installer_type = CONST.__getattribute__('INSTALLER_TYPE') deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO') - if (bool(installer_type) * bool(deploy_scenario)): - if 'scenario' in black_list_yaml.keys(): - for item in black_list_yaml['scenario']: - scenarios = item['scenarios'] - installers = item['installers'] - if (deploy_scenario in scenarios and - installer_type in installers): - tests = item['tests'] - black_tests.extend(tests) + if (bool(installer_type) and bool(deploy_scenario) and + 'scenario' in black_list_yaml.keys()): + for item in black_list_yaml['scenario']: + scenarios = item['scenarios'] + installers = item['installers'] + in_it = RallyBase.in_iterable_re + if (in_it(deploy_scenario, scenarios) and + in_it(installer_type, installers)): + tests = item['tests'] + black_tests.extend(tests) except Exception: - logger.debug("Scenario exclusion not applied.") + LOGGER.debug("Scenario exclusion not applied.") return black_tests @staticmethod + def in_iterable_re(needle, haystack): + """ + Check if given needle is in the iterable haystack, using regex. + + :param needle: string to be matched + :param haystack: iterable of strings (optionally regex patterns) + :return: True if needle is eqial to any of the elements in haystack, + or if a nonempty regex pattern in haystack is found in needle. + """ + # match without regex + if needle in haystack: + return True + + for pattern in haystack: + # match if regex pattern is set and found in the needle + if pattern and re.search(pattern, needle) is not None: + return True + else: + return False + + @staticmethod def excl_func(): + """Exclude functionalities.""" black_tests = [] func_list = [] @@ -221,19 +261,23 @@ class RallyBase(testcase.OSGCTestCase): if func in functions: tests = item['tests'] black_tests.extend(tests) - except Exception: - logger.debug("Functionality exclusion not applied.") + except Exception: # pylint: disable=broad-except + LOGGER.debug("Functionality exclusion not applied.") return black_tests @staticmethod def apply_blacklist(case_file_name, result_file_name): - logger.debug("Applying blacklist...") + """Apply blacklist.""" + LOGGER.debug("Applying blacklist...") cases_file = open(case_file_name, 'r') result_file = open(result_file_name, 'w') black_tests = list(set(RallyBase.excl_func() + - RallyBase.excl_scenario())) + RallyBase.excl_scenario())) + + if black_tests: + LOGGER.debug("Blacklisted tests: " + str(black_tests)) include = True for cases_line in cases_file: @@ -254,56 +298,58 @@ class RallyBase(testcase.OSGCTestCase): @staticmethod def file_is_empty(file_name): + """Determine is a file is empty.""" try: if os.stat(file_name).st_size > 0: return False - except: + except Exception: # pylint: disable=broad-except pass return True def _run_task(self, test_name): - logger.info('Starting test scenario "{}" ...'.format(test_name)) + """Run a task.""" + LOGGER.info('Starting test scenario "%s" ...', test_name) task_file = os.path.join(self.RALLY_DIR, 'task.yaml') if not os.path.exists(task_file): - logger.error("Task file '%s' does not exist." % task_file) - raise Exception("Task file '%s' does not exist." % task_file) + LOGGER.error("Task file '%s' does not exist.", task_file) + raise Exception("Task file '%s' does not exist.", task_file) file_name = self._prepare_test_list(test_name) if self.file_is_empty(file_name): - logger.info('No tests for scenario "{}"'.format(test_name)) + LOGGER.info('No tests for scenario "%s"', test_name) return cmd_line = ("rally task start --abort-on-sla-failure " "--task {0} " "--task-args \"{1}\"" .format(task_file, self._build_task_args(test_name))) - logger.debug('running command line: {}'.format(cmd_line)) + LOGGER.debug('running command line: %s', cmd_line) - p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, shell=True) - output = self._get_output(p, test_name) + proc = subprocess.Popen(cmd_line, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True) + output = self._get_output(proc, test_name) task_id = self.get_task_id(output) - logger.debug('task_id : {}'.format(task_id)) + LOGGER.debug('task_id : %s', task_id) if task_id is None: - logger.error('Failed to retrieve task_id, validating task...') + LOGGER.error('Failed to retrieve task_id, validating task...') cmd_line = ("rally task validate " "--task {0} " "--task-args \"{1}\"" .format(task_file, self._build_task_args(test_name))) - logger.debug('running command line: {}'.format(cmd_line)) - p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, shell=True) - output = self.get_cmd_output(p) - logger.error("Task validation result:" + "\n" + output) + LOGGER.debug('running command line: %s', cmd_line) + proc = subprocess.Popen(cmd_line, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True) + output = self.get_cmd_output(proc) + LOGGER.error("Task validation result:" + "\n" + output) return # check for result directory and create it otherwise if not os.path.exists(self.RESULTS_DIR): - logger.debug('{} does not exist, we create it.' - .format(self.RESULTS_DIR)) + LOGGER.debug('%s does not exist, we create it.', + self.RESULTS_DIR) os.makedirs(self.RESULTS_DIR) # write html report file @@ -312,25 +358,25 @@ class RallyBase(testcase.OSGCTestCase): cmd_line = "rally task report {} --out {}".format(task_id, report_html_dir) - logger.debug('running command line: {}'.format(cmd_line)) + LOGGER.debug('running command line: %s', cmd_line) os.popen(cmd_line) # get and save rally operation JSON result cmd_line = "rally task results %s" % task_id - logger.debug('running command line: {}'.format(cmd_line)) + LOGGER.debug('running command line: %s', cmd_line) cmd = os.popen(cmd_line) json_results = cmd.read() report_json_name = 'opnfv-{}.json'.format(test_name) report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name) - with open(report_json_dir, 'w') as f: - logger.debug('saving json file') - f.write(json_results) + with open(report_json_dir, 'w') as r_file: + LOGGER.debug('saving json file') + r_file.write(json_results) - """ parse JSON operation result """ + # parse JSON operation result if self.task_succeed(json_results): - logger.info('Test scenario: "{}" OK.'.format(test_name) + "\n") + LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n") else: - logger.info('Test scenario: "{}" Failed.'.format(test_name) + "\n") + LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n") def _get_output(self, proc, test_name): result = "" @@ -367,15 +413,15 @@ class RallyBase(testcase.OSGCTestCase): try: success += float(percentage) except ValueError: - logger.info('Percentage error: %s, %s' % - (percentage, line)) + LOGGER.info('Percentage error: %s, %s', + percentage, line) nb_totals += 1 elif "Full duration" in line: duration = line.split(': ')[1] try: overall_duration += float(duration) except ValueError: - logger.info('Duration error: %s, %s' % (duration, line)) + LOGGER.info('Duration error: %s, %s', duration, line) overall_duration = "{:10.2f}".format(overall_duration) if nb_totals == 0: @@ -389,30 +435,30 @@ class RallyBase(testcase.OSGCTestCase): 'success': success_avg} self.summary.append(scenario_summary) - logger.debug("\n" + result) + LOGGER.debug("\n" + result) return result def _prepare_env(self): - logger.debug('Validating the test name...') - if not (self.test_name in self.TESTS): + LOGGER.debug('Validating the test name...') + if self.test_name not in self.TESTS: raise Exception("Test name '%s' is invalid" % self.test_name) volume_types = os_utils.list_volume_types(self.cinder_client, private=False) if volume_types: - logger.debug("Using existing volume type(s)...") + LOGGER.debug("Using existing volume type(s)...") else: - logger.debug('Creating volume type...') + LOGGER.debug('Creating volume type...') self.volume_type = os_utils.create_volume_type( self.cinder_client, self.CINDER_VOLUME_TYPE_NAME) if self.volume_type is None: raise Exception("Failed to create volume type '%s'" % self.CINDER_VOLUME_TYPE_NAME) - logger.debug("Volume type '%s' is created succesfully." % + LOGGER.debug("Volume type '%s' is created succesfully.", self.CINDER_VOLUME_TYPE_NAME) - logger.debug('Getting or creating image...') + LOGGER.debug('Getting or creating image...') self.image_exists, self.image_id = os_utils.get_or_create_image( self.GLANCE_IMAGE_NAME, self.GLANCE_IMAGE_PATH, @@ -421,7 +467,7 @@ class RallyBase(testcase.OSGCTestCase): raise Exception("Failed to get or create image '%s'" % self.GLANCE_IMAGE_NAME) - logger.debug("Creating network '%s'..." % self.RALLY_PRIVATE_NET_NAME) + LOGGER.debug("Creating network '%s'...", self.RALLY_PRIVATE_NET_NAME) self.network_dict = os_utils.create_shared_network_full( self.RALLY_PRIVATE_NET_NAME, self.RALLY_PRIVATE_SUBNET_NAME, @@ -434,7 +480,7 @@ class RallyBase(testcase.OSGCTestCase): def _run_tests(self): if self.test_name == 'all': for test in self.TESTS: - if (test == 'all' or test == 'vm'): + if test == 'all' or test == 'vm': continue self._run_task(test) else: @@ -459,25 +505,25 @@ class RallyBase(testcase.OSGCTestCase): total_duration = 0.0 total_nb_tests = 0 total_success = 0.0 - for s in self.summary: - name = "{0:<17}".format(s['test_name']) - duration = float(s['overall_duration']) + for item in self.summary: + name = "{0:<17}".format(item['test_name']) + duration = float(item['overall_duration']) total_duration += duration duration = time.strftime("%M:%S", time.gmtime(duration)) duration = "{0:<10}".format(duration) - nb_tests = "{0:<13}".format(s['nb_tests']) - total_nb_tests += int(s['nb_tests']) - success = "{0:<10}".format(str(s['success']) + '%') - total_success += float(s['success']) + nb_tests = "{0:<13}".format(item['nb_tests']) + total_nb_tests += int(item['nb_tests']) + success = "{0:<10}".format(str(item['success']) + '%') + total_success += float(item['success']) report += ("" + "| " + name + " | " + duration + " | " + nb_tests + " | " + success + "|\n" + "+-------------------+------------" "+---------------+-----------+\n") payload.append({'module': name, - 'details': {'duration': s['overall_duration'], - 'nb tests': s['nb_tests'], - 'success': s['success']}}) + 'details': {'duration': item['overall_duration'], + 'nb tests': item['nb_tests'], + 'success': item['success']}}) total_duration_str = time.strftime("%H:%M:%S", time.gmtime(total_duration)) @@ -500,29 +546,31 @@ class RallyBase(testcase.OSGCTestCase): "+===============+===========+") report += "\n" - logger.info("\n" + report) + LOGGER.info("\n" + report) payload.append({'summary': {'duration': total_duration, 'nb tests': total_nb_tests, 'nb success': success_rate}}) self.details = payload - logger.info("Rally '%s' success_rate is %s%%" - % (self.case_name, success_rate)) + LOGGER.info("Rally '%s' success_rate is %s%%", + self.case_name, success_rate) def _clean_up(self): if self.volume_type: - logger.debug("Deleting volume type '%s'..." % self.volume_type) + LOGGER.debug("Deleting volume type '%s'...", self.volume_type) os_utils.delete_volume_type(self.cinder_client, self.volume_type) if not self.image_exists: - logger.debug("Deleting image '%s' with ID '%s'..." - % (self.GLANCE_IMAGE_NAME, self.image_id)) + LOGGER.debug("Deleting image '%s' with ID '%s'...", + self.GLANCE_IMAGE_NAME, self.image_id) if not os_utils.delete_glance_image(self.nova_client, self.image_id): - logger.error("Error deleting the glance image") + LOGGER.error("Error deleting the glance image") - def run(self): + @energy.enable_recording + def run(self, **kwargs): + """Run testcase.""" self.start_time = time.time() try: self._prepare_env() @@ -530,8 +578,8 @@ class RallyBase(testcase.OSGCTestCase): self._generate_report() self._clean_up() res = testcase.TestCase.EX_OK - except Exception as e: - logger.error('Error with run: %s' % e) + except Exception as exc: # pylint: disable=broad-except + LOGGER.error('Error with run: %s', exc) res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() @@ -539,7 +587,10 @@ class RallyBase(testcase.OSGCTestCase): class RallySanity(RallyBase): + """Rally sanity testcase implementation.""" + def __init__(self, **kwargs): + """Initialize RallySanity object.""" if "case_name" not in kwargs: kwargs["case_name"] = "rally_sanity" super(RallySanity, self).__init__(**kwargs) @@ -550,7 +601,10 @@ class RallySanity(RallyBase): class RallyFull(RallyBase): + """Rally full testcase implementation.""" + def __init__(self, **kwargs): + """Initialize RallyFull object.""" if "case_name" not in kwargs: kwargs["case_name"] = "rally_full" super(RallyFull, self).__init__(**kwargs) diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py index c2a05379d..6ac721762 100644 --- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py +++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py @@ -6,59 +6,69 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 +"""Refstack client testcase implemenation.""" + from __future__ import division import argparse import logging import os -import pkg_resources import re import sys import subprocess import time +import pkg_resources + from functest.core import testcase +from functest.energy import energy +from functest.opnfv_tests.openstack.refstack_client.tempest_conf \ + import TempestConf from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils.constants import CONST import functest.utils.functest_utils as ft_utils -from tempest_conf import TempestConf -""" logging configuration """ -logger = logging.getLogger(__name__) +# logging configuration """ +LOGGER = logging.getLogger(__name__) class RefstackClient(testcase.OSGCTestCase): + """RefstackClient testcase implementation class.""" def __init__(self, **kwargs): + """Initialize RefstackClient testcase object.""" if "case_name" not in kwargs: kwargs["case_name"] = "refstack_defcore" super(RefstackClient, self).__init__(**kwargs) - self.CONF_PATH = pkg_resources.resource_filename( + self.conf_path = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/refstack_client/refstack_tempest.conf') - self.FUNCTEST_TEST = pkg_resources.resource_filename( + self.functest_test = pkg_resources.resource_filename( 'functest', 'opnfv_tests') - self.DEFCORE_LIST = 'openstack/refstack_client/defcore.txt' - self.confpath = os.path.join(self.FUNCTEST_TEST, - self.CONF_PATH) + self.defcore_list = 'openstack/refstack_client/defcore.txt' + self.confpath = os.path.join(self.functest_test, + self.conf_path) self.defcorelist = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/refstack_client/defcore.txt') + self.testlist = None self.insecure = '' if ('https' in CONST.__getattribute__('OS_AUTH_URL') and CONST.__getattribute__('OS_INSECURE').lower() == 'true'): self.insecure = '-k' def run_defcore(self, conf, testlist): + """Run defcore sys command.""" cmd = ("refstack-client test {0} -c {1} -v --test-list {2}" .format(self.insecure, conf, testlist)) - logger.info("Starting Refstack_defcore test case: '%s'." % cmd) + LOGGER.info("Starting Refstack_defcore test case: '%s'.", cmd) ft_utils.execute_command(cmd) def run_defcore_default(self): + """Run default defcare sys command.""" cmd = ("refstack-client test {0} -c {1} -v --test-list {2}" .format(self.insecure, self.confpath, self.defcorelist)) - logger.info("Starting Refstack_defcore test case: '%s'." % cmd) + LOGGER.info("Starting Refstack_defcore test case: '%s'.", cmd) header = ("Refstack environment:\n" " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % @@ -74,40 +84,41 @@ class RefstackClient(testcase.OSGCTestCase): "environment.log"), 'w+') f_env.write(header) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, bufsize=1) + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, bufsize=1) - with p.stdout: - for line in iter(p.stdout.readline, b''): + with process.stdout: + for line in iter(process.stdout.readline, b''): if 'Tests' in line: break - if re.search("\} tempest\.", line): - logger.info(line.replace('\n', '')) + if re.search(r"\} tempest\.", line): + LOGGER.info(line.replace('\n', '')) f_stdout.write(line) - p.wait() + process.wait() f_stdout.close() f_env.close() def parse_refstack_result(self): + """Parse Refstact results.""" try: with open(os.path.join(conf_utils.REFSTACK_RESULTS_DIR, "refstack.log"), 'r') as logfile: output = logfile.read() - for match in re.findall("Ran: (\d+) tests in (\d+\.\d{4}) sec.", + for match in re.findall(r"Ran: (\d+) tests in (\d+\.\d{4}) sec.", output): num_tests = match[0] - logger.info("Ran: %s tests in %s sec." % (num_tests, match[1])) - for match in re.findall("(- Passed: )(\d+)", output): + LOGGER.info("Ran: %s tests in %s sec.", num_tests, match[1]) + for match in re.findall(r"(- Passed: )(\d+)", output): num_success = match[1] - logger.info("".join(match)) - for match in re.findall("(- Skipped: )(\d+)", output): + LOGGER.info("".join(match)) + for match in re.findall(r"(- Skipped: )(\d+)", output): num_skipped = match[1] - logger.info("".join(match)) - for match in re.findall("(- Failed: )(\d+)", output): + LOGGER.info("".join(match)) + for match in re.findall(r"(- Failed: )(\d+)", output): num_failures = match[1] - logger.info("".join(match)) + LOGGER.info("".join(match)) success_testcases = "" for match in re.findall(r"\{0\}(.*?)[. ]*ok", output): success_testcases += match + ", " @@ -123,7 +134,7 @@ class RefstackClient(testcase.OSGCTestCase): try: self.result = 100 * int(num_success) / int(num_executed) except ZeroDivisionError: - logger.error("No test has been executed") + LOGGER.error("No test has been executed") self.details = {"tests": int(num_tests), "failures": int(num_failures), @@ -133,12 +144,17 @@ class RefstackClient(testcase.OSGCTestCase): except Exception: self.result = 0 - logger.info("Testcase %s success_rate is %s%%" - % (self.case_name, self.result)) + LOGGER.info("Testcase %s success_rate is %s%%", + self.case_name, self.result) - def run(self): - '''used for functest command line, - functest testcase run refstack_defcore''' + @energy.enable_recording + def run(self, **kwargs): + """ + Start RefstackClient testcase. + + used for functest command line, + functest testcase run refstack_defcore + """ self.start_time = time.time() if not os.path.exists(conf_utils.REFSTACK_RESULTS_DIR): @@ -150,59 +166,64 @@ class RefstackClient(testcase.OSGCTestCase): self.run_defcore_default() self.parse_refstack_result() res = testcase.TestCase.EX_OK - except Exception as e: - logger.error('Error with run: %s', e) + except Exception: + LOGGER.exception("Error with run") res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() return res def _prep_test(self): - '''Check that the config file exists.''' + """Check that the config file exists.""" if not os.path.isfile(self.confpath): - logger.error("Conf file not valid: %s" % self.confpath) + LOGGER.error("Conf file not valid: %s", self.confpath) if not os.path.isfile(self.testlist): - logger.error("testlist file not valid: %s" % self.testlist) + LOGGER.error("testlist file not valid: %s", self.testlist) def main(self, **kwargs): - '''used for manually running, + """ + Execute RefstackClient testcase manually. + + used for manually running, python refstack_client.py -c <tempest_conf_path> --testlist <testlist_path> can generate a reference refstack_tempest.conf by python tempest_conf.py - ''' + """ try: self.confpath = kwargs['config'] self.testlist = kwargs['testlist'] - except KeyError as e: - logger.error("Cannot run refstack client. Please check " - "%s", e) + except KeyError as exc: + LOGGER.error("Cannot run refstack client. Please check " + "%s", exc) return self.EX_RUN_ERROR try: self._prep_test() self.run_defcore(self.confpath, self.testlist) res = testcase.TestCase.EX_OK - except Exception as e: - logger.error('Error with run: %s', e) + except Exception as exc: + LOGGER.error('Error with run: %s', exc) res = testcase.TestCase.EX_RUN_ERROR return res -class RefstackClientParser(object): +class RefstackClientParser(object): # pylint: disable=too-few-public-methods + """Command line argument parser helper.""" def __init__(self): - self.FUNCTEST_TEST = pkg_resources.resource_filename( + """Initialize helper object.""" + self.functest_test = pkg_resources.resource_filename( 'functest', 'opnfv_tests') - self.CONF_PATH = pkg_resources.resource_filename( + self.conf_path = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/refstack_client/refstack_tempest.conf') - self.DEFCORE_LIST = pkg_resources.resource_filename( + self.defcore_list = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/refstack_client/defcore.txt') - self.confpath = os.path.join(self.FUNCTEST_TEST, - self.CONF_PATH) - self.defcorelist = os.path.join(self.FUNCTEST_TEST, - self.DEFCORE_LIST) + self.confpath = os.path.join(self.functest_test, + self.conf_path) + self.defcorelist = os.path.join(self.functest_test, + self.defcore_list) self.parser = argparse.ArgumentParser() self.parser.add_argument( '-c', '--config', @@ -215,11 +236,13 @@ class RefstackClientParser(object): 'should be tested.', default=self.defcorelist) - def parse_args(self, argv=[]): + def parse_args(self, argv=None): + """Parse command line arguments.""" return vars(self.parser.parse_args(argv)) def main(): + """Run RefstackClient testcase with CLI.""" logging.basicConfig() refstackclient = RefstackClient() parser = RefstackClientParser() diff --git a/functest/opnfv_tests/openstack/vping/vping_ssh.py b/functest/opnfv_tests/openstack/vping/vping_ssh.py index d4c39ad27..5cacddb55 100644 --- a/functest/opnfv_tests/openstack/vping/vping_ssh.py +++ b/functest/opnfv_tests/openstack/vping/vping_ssh.py @@ -7,13 +7,25 @@ # # http://www.apache.org/licenses/LICENSE-2.0 -import os -import pkg_resources -from scp import SCPClient + +"""vPingSSH testcase.""" + +# This 1st import is here simply for pep8 as the 'os' package import appears +# to be required for mock and the unit tests will fail without it +import os # noqa # pylint: disable=unused-import import time +from scp import SCPClient +import pkg_resources + +from functest.core.testcase import TestCase +from functest.energy import energy +from functest.opnfv_tests.openstack.snaps import snaps_utils +from functest.opnfv_tests.openstack.vping import vping_base +from functest.utils.constants import CONST from snaps.openstack.create_instance import FloatingIpSettings, \ VmInstanceSettings + from snaps.openstack.create_keypairs import KeypairSettings from snaps.openstack.create_network import PortSettings from snaps.openstack.create_router import RouterSettings @@ -21,24 +33,17 @@ from snaps.openstack.create_security_group import Direction, Protocol, \ SecurityGroupSettings, SecurityGroupRuleSettings from snaps.openstack.utils import deploy_utils -from functest.core.testcase import TestCase -from functest.opnfv_tests.openstack.snaps import snaps_utils -from functest.opnfv_tests.openstack.vping import vping_base -from functest.utils.constants import CONST - class VPingSSH(vping_base.VPingBase): """ + VPingSSH testcase implementation. + Class to execute the vPing test using a Floating IP to connect to one VM to issue the ping command to the second """ def __init__(self, **kwargs): - - # This line is here simply for pep8 as the 'os' package import appears - # to be required for mock and the unit tests will fail without it - os.environ - + """Initialize testcase.""" if "case_name" not in kwargs: kwargs["case_name"] = "vping_ssh" super(VPingSSH, self).__init__(**kwargs) @@ -51,8 +56,11 @@ class VPingSSH(vping_base.VPingBase): self.sg_name = CONST.__getattribute__('vping_sg_name') + self.guid self.sg_desc = CONST.__getattribute__('vping_sg_desc') + @energy.enable_recording def run(self): """ + Excecute VPingSSH testcase. + Sets up the OpenStack keypair, router, security group, and VM instance objects then validates the ping. :return: the exit code from the super.execute() method @@ -60,7 +68,8 @@ class VPingSSH(vping_base.VPingBase): try: super(VPingSSH, self).run() - self.logger.info("Creating keypair with name: '%s'" % self.kp_name) + log = "Creating keypair with name: '%s'" % self.kp_name + self.logger.info(log) kp_creator = deploy_utils.create_keypair( self.os_creds, KeypairSettings(name=self.kp_name, @@ -69,8 +78,8 @@ class VPingSSH(vping_base.VPingBase): self.creators.append(kp_creator) # Creating router to external network - self.logger.info("Creating router with name: '%s'" - % self.router_name) + log = "Creating router with name: '%s'" % self.router_name + self.logger.info(log) net_set = self.network_creator.network_settings sub_set = [net_set.subnet_settings[0].name] ext_net_name = snaps_utils.get_ext_net_name(self.os_creds) @@ -93,9 +102,9 @@ class VPingSSH(vping_base.VPingBase): ssh_connect_timeout=self.vm_ssh_connect_timeout, port_settings=[port1_settings]) - self.logger.info( - "Creating VM 1 instance with name: '%s'" - % instance1_settings.name) + log = ("Creating VM 1 instance with name: '%s'" + % instance1_settings.name) + self.logger.info(log) self.vm1_creator = deploy_utils.create_vm_instance( self.os_creds, instance1_settings, @@ -122,9 +131,9 @@ class VPingSSH(vping_base.VPingBase): port_name=port2_settings.name, router_name=router_creator.router_settings.name)]) - self.logger.info( - "Creating VM 2 instance with name: '%s'" - % instance2_settings.name) + log = ("Creating VM 2 instance with name: '%s'" + % instance2_settings.name) + self.logger.info(log) self.vm2_creator = deploy_utils.create_vm_instance( self.os_creds, instance2_settings, @@ -133,14 +142,16 @@ class VPingSSH(vping_base.VPingBase): self.creators.append(self.vm2_creator) return self._execute() - except Exception as e: - self.logger.error('Unexpected error running test - ' + e.message) + except Exception as exc: # pylint: disable=broad-except + self.logger.error('Unexpected error running test - ' + exc.message) return TestCase.EX_RUN_ERROR finally: self._cleanup() def _do_vping(self, vm_creator, test_ip): """ + Execute ping command. + Override from super """ if vm_creator.vm_ssh_active(block=True): @@ -153,6 +164,8 @@ class VPingSSH(vping_base.VPingBase): def _transfer_ping_script(self, ssh): """ + Transfert vping script to VM. + Uses SCP to copy the ping script via the SSH client :param ssh: the SSH client :return: @@ -163,11 +176,12 @@ class VPingSSH(vping_base.VPingBase): 'functest.opnfv_tests.openstack.vping', 'ping.sh') try: scp.put(ping_script, "~/") - except: - self.logger.error("Cannot SCP the file '%s'" % ping_script) + except Exception: + self.logger.error("Cannot SCP the file '%s'", ping_script) return False cmd = 'chmod 755 ~/ping.sh' + # pylint: disable=unused-variable (stdin, stdout, stderr) = ssh.exec_command(cmd) for line in stdout.readlines(): print line @@ -176,6 +190,8 @@ class VPingSSH(vping_base.VPingBase): def _do_vping_ssh(self, ssh, test_ip): """ + Execute ping command via SSH. + Pings the test_ip via the SSH client :param ssh: the SSH client used to issue the ping command :param test_ip: the IP for the ping command to use @@ -190,7 +206,7 @@ class VPingSSH(vping_base.VPingBase): while True: time.sleep(1) - (stdin, stdout, stderr) = ssh.exec_command(cmd) + (_, stdout, _) = ssh.exec_command(cmd) output = stdout.readlines() for line in output: @@ -206,12 +222,15 @@ class VPingSSH(vping_base.VPingBase): break if flag: break - self.logger.debug("Pinging %s. Waiting for response..." % test_ip) + log = "Pinging %s. Waiting for response..." % test_ip + self.logger.debug(log) sec += 1 return exit_code def __create_security_group(self): """ + Configure OpenStack security groups. + Configures and deploys an OpenStack security group object :return: the creator object """ @@ -231,7 +250,8 @@ class VPingSSH(vping_base.VPingBase): protocol=Protocol.tcp, port_range_min=22, port_range_max=22)) - self.logger.info("Security group with name: '%s'" % self.sg_name) + log = "Security group with name: '%s'" % self.sg_name + self.logger.info(log) return deploy_utils.create_security_group(self.os_creds, SecurityGroupSettings( name=self.sg_name, diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py index 2dcce4084..fafc77e13 100644 --- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py +++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py @@ -7,15 +7,19 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 +"""CloudifyIms testcase implementation.""" + import logging import os import time -import yaml -from scp import SCPClient from cloudify_rest_client import CloudifyClient from cloudify_rest_client.executions import Execution +from scp import SCPClient +import yaml +from functest.energy import energy +from functest.opnfv_tests.openstack.snaps import snaps_utils import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base from functest.utils.constants import CONST import functest.utils.openstack_utils as os_utils @@ -36,18 +40,17 @@ from snaps.openstack.create_image import ImageSettings, OpenStackImage from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair from snaps.openstack.create_network import PortSettings -from functest.opnfv_tests.openstack.snaps import snaps_utils - __author__ = "Valentin Boucher <valentin.boucher@orange.com>" class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): - """Clearwater vIMS deployed with Cloudify Orchestrator Case""" + """Clearwater vIMS deployed with Cloudify Orchestrator Case.""" __logger = logging.getLogger(__name__) def __init__(self, **kwargs): + """Initialize CloudifyIms testcase object.""" if "case_name" not in kwargs: kwargs["case_name"] = "cloudify_ims" super(CloudifyIms, self).__init__(**kwargs) @@ -93,6 +96,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): self.__logger.info("Images needed for vIMS: %s", self.images) def prepare(self): + """Prepare testscase (Additional pre-configuration steps).""" super(CloudifyIms, self).prepare() self.__logger.info("Additional pre-configuration steps") @@ -120,7 +124,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): def deploy_orchestrator(self): """ - Deploy Cloudify Manager + Deploy Cloudify Manager. network, security group, fip, VM creation """ @@ -277,9 +281,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): return True def deploy_vnf(self): - """ - Deploy Clearwater IMS - """ + """Deploy Clearwater IMS.""" start_time = time.time() self.__logger.info("Upload VNFD") @@ -323,15 +325,14 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): self.__logger.info(execution) if execution.status == 'terminated': self.details['vnf'].update(status='PASS', duration=duration) - return True + result = True else: self.details['vnf'].update(status='FAIL', duration=duration) - return False + result = False + return result def test_vnf(self): - """ - Run test on clearwater ims instance - """ + """Run test on clearwater ims instance.""" start_time = time.time() cfy_client = self.orchestrator['object'] @@ -342,22 +343,23 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): ellis_ip = outputs['ellis_ip'] self.config_ellis(ellis_ip) - if dns_ip != "": - vims_test_result = self.run_clearwater_live_test( - dns_ip=dns_ip, - public_domain=self.vnf['inputs']["public_domain"]) - duration = time.time() - start_time - short_result = sig_test_format(vims_test_result) - self.__logger.info(short_result) - self.details['test_vnf'].update(status='PASS', - result=short_result, - full_result=vims_test_result, - duration=duration) - return True - else: + if not dns_ip: return False + vims_test_result = self.run_clearwater_live_test( + dns_ip=dns_ip, + public_domain=self.vnf['inputs']["public_domain"]) + duration = time.time() - start_time + short_result = sig_test_format(vims_test_result) + self.__logger.info(short_result) + self.details['test_vnf'].update(status='PASS', + result=short_result, + full_result=vims_test_result, + duration=duration) + return True + def clean(self): + """Clean created objects/functions.""" try: cfy_client = self.orchestrator['object'] dep_name = self.vnf['descriptor'].get('name') @@ -389,10 +391,15 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): for creator in reversed(self.created_object): try: creator.clean() - except Exception as e: - self.logger.error('Unexpected error cleaning - %s', e) + except Exception as exc: + self.logger.error('Unexpected error cleaning - %s', exc) super(CloudifyIms, self).clean() + @energy.enable_recording + def run(self, **kwargs): + """Execute CloudifyIms test case.""" + super(CloudifyIms, self).run(**kwargs) + # ---------------------------------------------------------- # @@ -401,6 +408,8 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): # ----------------------------------------------------------- def get_config(parameter, file_path): """ + Get config parameter. + Returns the value of a given parameter in file.yaml parameter must be given in string format with dots Example: general.openstack.image_name @@ -418,9 +427,7 @@ def get_config(parameter, file_path): def wait_for_execution(client, execution, logger, timeout=2400, ): - """ - Wait for a workflow execution on Cloudify Manager - """ + """Wait for a workflow execution on Cloudify Manager.""" # if execution already ended - return without waiting if execution.status in Execution.END_STATES: return execution @@ -470,7 +477,7 @@ def wait_for_execution(client, execution, logger, timeout=2400, ): def _get_deployment_environment_creation_execution(client, deployment_id): """ - Get the execution id of a env preparation + Get the execution id of a env preparation. network, security group, fip, VM creation """ @@ -484,9 +491,7 @@ def _get_deployment_environment_creation_execution(client, deployment_id): def sig_test_format(sig_test): - """ - Process the signaling result to have a short result - """ + """Process the signaling result to have a short result.""" nb_passed = 0 nb_failures = 0 nb_skipped = 0 diff --git a/functest/tests/unit/energy/test_functest_energy.py b/functest/tests/unit/energy/test_functest_energy.py index 177788bc4..f8bb13c99 100644 --- a/functest/tests/unit/energy/test_functest_energy.py +++ b/functest/tests/unit/energy/test_functest_energy.py @@ -248,7 +248,9 @@ class EnergyRecorderTest(unittest.TestCase): self.__decorated_method() == self.returned_value_to_preserve ) - def test_decorator_preserve_ex(self): + @mock.patch( + "functest.energy.energy.finish_session") + def test_decorator_preserve_ex(self, finish_mock=None): """Test that decorator preserve method exceptions.""" self.test_load_config() with self.assertRaises(Exception) as context: @@ -256,6 +258,7 @@ class EnergyRecorderTest(unittest.TestCase): self.assertTrue( self.exception_message_to_preserve in context.exception ) + self.assertTrue(finish_mock.called) @mock.patch("functest.utils.functest_utils.get_functest_config", side_effect=config_loader_mock) diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index 8845f6604..32cc15134 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -130,7 +130,10 @@ class OSRallyTesting(unittest.TestCase): CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario') dic = {'scenario': [{'scenarios': ['test_scenario'], 'installers': ['test_installer'], - 'tests': ['test']}]} + 'tests': ['test']}, + {'scenarios': ['other_scenario'], + 'installers': ['test_installer'], + 'tests': ['other_test']}]} with mock.patch('__builtin__.open', mock.mock_open()), \ mock.patch('functest.opnfv_tests.openstack.rally.rally.' 'yaml.safe_load', @@ -138,6 +141,34 @@ class OSRallyTesting(unittest.TestCase): self.assertEqual(self.rally_base.excl_scenario(), ['test']) + def test_excl_scenario_regex(self): + CONST.__setattr__('INSTALLER_TYPE', 'test_installer') + CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT') + dic = {'scenario': [{'scenarios': ['^os-[^-]+-featT-modeT$'], + 'installers': ['test_installer'], + 'tests': ['test1']}, + {'scenarios': ['^os-ctrlT-[^-]+-modeT$'], + 'installers': ['test_installer'], + 'tests': ['test2']}, + {'scenarios': ['^os-ctrlT-featT-[^-]+$'], + 'installers': ['test_installer'], + 'tests': ['test3']}, + {'scenarios': ['^os-'], + 'installers': ['test_installer'], + 'tests': ['test4']}, + {'scenarios': ['other_scenario'], + 'installers': ['test_installer'], + 'tests': ['test0a']}, + {'scenarios': [''], # empty scenario + 'installers': ['test_installer'], + 'tests': ['test0b']}]} + with mock.patch('__builtin__.open', mock.mock_open()), \ + mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'yaml.safe_load', + return_value=dic): + self.assertEqual(self.rally_base.excl_scenario(), + ['test1', 'test2', 'test3', 'test4']) + def test_excl_scenario_exception(self): with mock.patch('__builtin__.open', side_effect=Exception): self.assertEqual(self.rally_base.excl_scenario(), @@ -186,7 +217,7 @@ class OSRallyTesting(unittest.TestCase): self.assertRaises(Exception): self.rally_base._run_task('test_name') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info') def test_run_task_no_tests_for_scenario(self, mock_logger_info): with mock.patch('functest.opnfv_tests.openstack.rally.rally.' 'os.path.exists', @@ -196,10 +227,10 @@ class OSRallyTesting(unittest.TestCase): mock.patch.object(self.rally_base, 'file_is_empty', return_value=True): self.rally_base._run_task('test_name') - str = 'No tests for scenario "test_name"' - mock_logger_info.assert_any_call(str) + mock_logger_info.assert_any_call('No tests for scenario \"%s\"', + 'test_name') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.error') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error') def test_run_task_taskid_missing(self, mock_logger_error): with mock.patch('functest.opnfv_tests.openstack.rally.rally.' 'os.path.exists', @@ -222,8 +253,8 @@ class OSRallyTesting(unittest.TestCase): str = 'Failed to retrieve task_id, validating task...' mock_logger_error.assert_any_call(str) - @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.error') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error') def test_run_task_default(self, mock_logger_error, mock_logger_info): popen = mock.Mock() diff --git a/functest/utils/openstack_utils.py b/functest/utils/openstack_utils.py index f8719bf0c..4f8d6c357 100644 --- a/functest/utils/openstack_utils.py +++ b/functest/utils/openstack_utils.py @@ -175,11 +175,11 @@ def get_session_auth(other_creds={}): return auth -def get_endpoint(service_type, endpoint_type='publicURL'): +def get_endpoint(service_type, interface='public'): auth = get_session_auth() return get_session().get_endpoint(auth=auth, service_type=service_type, - endpoint_type=endpoint_type) + interface=interface) def get_session(other_creds={}): |