diff options
Diffstat (limited to 'testcases')
-rwxr-xr-x | testcases/Controllers/ODL/OpenDaylightTesting.py | 4 | ||||
-rw-r--r-- | testcases/Controllers/ONOS/Sfc/Sfc_fun.py | 16 | ||||
-rwxr-xr-x | testcases/OpenStack/healthcheck/healthcheck.sh | 5 | ||||
-rw-r--r-- | testcases/OpenStack/tempest/custom_tests/blacklist.txt | 34 | ||||
-rwxr-xr-x | testcases/OpenStack/tempest/gen_tempest_conf.py | 4 | ||||
-rwxr-xr-x | testcases/OpenStack/tempest/run_tempest.py | 5 | ||||
-rwxr-xr-x | testcases/features/copper.py | 171 | ||||
-rwxr-xr-x | testcases/features/doctor.py | 44 | ||||
-rwxr-xr-x | testcases/features/domino.py | 48 | ||||
-rwxr-xr-x | testcases/features/sfc/set-up-tacker.sh | 6 | ||||
-rwxr-xr-x | testcases/features/sfc/sfc.py | 18 | ||||
-rw-r--r-- | testcases/security_scan/scripts/internet_check.py | 25 | ||||
-rwxr-xr-x | testcases/security_scan/security_scan.py | 52 | ||||
-rw-r--r-- | testcases/vnf/vRNC/parser.py | 5 |
14 files changed, 276 insertions, 161 deletions
diff --git a/testcases/Controllers/ODL/OpenDaylightTesting.py b/testcases/Controllers/ODL/OpenDaylightTesting.py index db7b94982..79e385a2d 100755 --- a/testcases/Controllers/ODL/OpenDaylightTesting.py +++ b/testcases/Controllers/ODL/OpenDaylightTesting.py @@ -6,9 +6,11 @@ import os import re import shutil import sys -import functest.utils.functest_logger as ft_logger + from robot import run +import functest.utils.functest_logger as ft_logger + class ODLTestCases: diff --git a/testcases/Controllers/ONOS/Sfc/Sfc_fun.py b/testcases/Controllers/ONOS/Sfc/Sfc_fun.py index b94eedcaf..0d9eaf80d 100644 --- a/testcases/Controllers/ONOS/Sfc/Sfc_fun.py +++ b/testcases/Controllers/ONOS/Sfc/Sfc_fun.py @@ -227,6 +227,22 @@ class Sfc_fun: else: return(response.status_code) + url = 'http://' + self.nova_hostname + \ + ':8774/v2.1/' + self.tenant_id + '/flavors?name=m1.tiny' + headers = {"Accept": "application/json", "Content-Type": + "application/json", "X-Auth-Token": self.token_id} + response = requests.get(url, headers=headers) + + if (response.status_code == 200): + self.logger.debug(response.status_code) + self.logger.debug(response.content) + self.logger.info("\tFlavor is available") + json1_data = json.loads(response.content) + self.logger.debug(json1_data) + self.flavorRef = json1_data['flavors'][0]['id'] + else: + return(response.status_code) + for y in range(0, 3): Dicdata = {} org_nw_port = [] diff --git a/testcases/OpenStack/healthcheck/healthcheck.sh b/testcases/OpenStack/healthcheck/healthcheck.sh index 2449ac93b..996aadcf7 100755 --- a/testcases/OpenStack/healthcheck/healthcheck.sh +++ b/testcases/OpenStack/healthcheck/healthcheck.sh @@ -120,6 +120,7 @@ info "...Keystone OK!" info "Testing Glance API..." ################################# disk_img=$(cat ${YAML_FILE} | shyaml get-value healthcheck.disk_image 2> /dev/null || true) +disk_format=$(cat ${YAML_FILE} | shyaml get-value healthcheck.disk_format 2> /dev/null || true) kernel_img=$(cat ${YAML_FILE} | shyaml get-value healthcheck.kernel_image 2> /dev/null || true) ramdisk_img=$(cat ${YAML_FILE} | shyaml get-value healthcheck.ramdisk_image 2> /dev/null || true) extra_properties=$(cat ${YAML_FILE} | shyaml get-value healthcheck.extra_properties 2> /dev/null || true) @@ -151,10 +152,10 @@ fi debug "image extra_properties=${extra_properties}" -eval glance image-create --name ${image_1} --disk-format qcow2 --container-format bare \ +eval glance image-create --name ${image_1} --disk-format ${disk_format} --container-format bare \ ${extra_opts} < ${disk_img} debug "image '${image_1}' created." -eval glance image-create --name ${image_2} --disk-format qcow2 --container-format bare \ +eval glance image-create --name ${image_2} --disk-format ${disk_format} --container-format bare \ ${extra_opts} < ${disk_img} debug "image '${image_2}' created." info "... Glance OK!" diff --git a/testcases/OpenStack/tempest/custom_tests/blacklist.txt b/testcases/OpenStack/tempest/custom_tests/blacklist.txt index 5dd8fe2e1..6dd7fad5c 100644 --- a/testcases/OpenStack/tempest/custom_tests/blacklist.txt +++ b/testcases/OpenStack/tempest/custom_tests/blacklist.txt @@ -16,18 +16,48 @@ - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern +- scenarios: - os-odl_l2-nofeature-ha - os-odl_l2-nofeature-noha - os-nosdn-nofeature-ha - os-nosdn-nofeature-noha + - os-nosdn-lxd-ha + - os-nosdn-lxd-noha + installers: + - joid + tests: + - tempest.api.object_storage + +- + scenarios: + - os-onos-nofeature-ha + - os-onos-nofeature-noha + - os-onos-sfc-ha + - os-onos-sfc-noha + installers: + - fuel + - apex + - compass + tests: + - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard + - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops + - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern + +- + scenarios: - os-onos-nofeature-ha - os-onos-nofeature-noha - os-onos-sfc-ha - os-onos-sfc-noha - - os-nosdn-lxd-ha - - os-nosdn-lxd-noha installers: - joid tests: - tempest.api.object_storage + - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard + - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops + - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern diff --git a/testcases/OpenStack/tempest/gen_tempest_conf.py b/testcases/OpenStack/tempest/gen_tempest_conf.py index 79236b958..4aa814a51 100755 --- a/testcases/OpenStack/tempest/gen_tempest_conf.py +++ b/testcases/OpenStack/tempest/gen_tempest_conf.py @@ -17,6 +17,7 @@ import shutil import functest.utils.functest_utils as ft_utils import functest.utils.functest_logger as ft_logger from run_tempest import configure_tempest +from run_tempest import TEMPEST_RESULTS_DIR logger = ft_logger.Logger("multisite").getLogger() @@ -112,6 +113,9 @@ def configure_tempest_multisite(deployment_dir): def main(): + if not os.path.exists(TEMPEST_RESULTS_DIR): + os.makedirs(TEMPEST_RESULTS_DIR) + deployment_dir = ft_utils.get_deployment_dir(logger) configure_tempest_multisite(deployment_dir) diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py index 3a8d8d72a..c143fffae 100755 --- a/testcases/OpenStack/tempest/run_tempest.py +++ b/testcases/OpenStack/tempest/run_tempest.py @@ -365,6 +365,8 @@ def run_tempest(OPTION): if 'smoke' in args.mode: case_name = 'tempest_smoke_serial' + elif 'feature' in args.mode: + case_name = args.mode.replace("feature_", "") else: case_name = 'tempest_full_parallel' @@ -422,7 +424,8 @@ def main(): deployment_dir = ft_utils.get_deployment_dir(logger) - if "" != args.conf: + if "" == args.conf: + MODE = "" configure_tempest(deployment_dir) else: MODE = " --tempest-config " + args.conf diff --git a/testcases/features/copper.py b/testcases/features/copper.py index 9efcbd7f4..c312643eb 100755 --- a/testcases/features/copper.py +++ b/testcases/features/copper.py @@ -1,86 +1,85 @@ -#!/usr/bin/python
-#
-# Copyright 2016 AT&T Intellectual Property, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import sys
-import time
-import functest.utils.functest_logger as ft_logger
-import functest.utils.functest_utils as functest_utils
-import yaml
-
-
-with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
- functest_yaml = yaml.safe_load(f)
-
-dirs = functest_yaml.get('general').get('directories')
-FUNCTEST_REPO = dirs.get('dir_repo_functest')
-COPPER_REPO = dirs.get('dir_repo_copper')
-TEST_DB_URL = functest_yaml.get('results').get('test_db_url')
-
-logger = ft_logger.Logger("copper").getLogger()
-
-
-def main():
- cmd = "%s/tests/run.sh %s/tests" % (COPPER_REPO, COPPER_REPO)
-
- start_time = time.time()
-
- ret_val = functest_utils.execute_command(cmd, logger, exit_on_error=False)
-
- stop_time = time.time()
- duration = round(stop_time - start_time, 1)
- if ret_val == 0:
- logger.info("COPPER PASSED")
- test_status = 'PASS'
- else:
- logger.info("COPPER FAILED")
- test_status = 'FAIL'
-
- details = {
- 'timestart': start_time,
- 'duration': duration,
- 'status': test_status,
- }
- pod_name = functest_utils.get_pod_name(logger)
- scenario = functest_utils.get_scenario(logger)
- version = functest_utils.get_version(logger)
- build_tag = functest_utils.get_build_tag(logger)
-
- logger.info("Pushing COPPER results: TEST_DB_URL=%(db)s pod_name=%(pod)s "
- "version=%(v)s scenario=%(s)s criteria=%(c)s details=%(d)s" % {
- 'db': TEST_DB_URL,
- 'pod': pod_name,
- 'v': version,
- 's': scenario,
- 'c': details['status'],
- 'b': build_tag,
- 'd': details,
- })
- functest_utils.push_results_to_db("COPPER",
- "COPPER-notification",
- logger,
- start_time,
- stop_time,
- details['status'],
- details)
- if ret_val != 0:
- sys.exit(-1)
-
- sys.exit(0)
-
-if __name__ == '__main__':
- main()
+#!/usr/bin/python +# +# Copyright 2016 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import sys +import time +import functest.utils.functest_logger as ft_logger +import functest.utils.functest_utils as functest_utils +import yaml + +parser = argparse.ArgumentParser() +parser.add_argument("-r", "--report", + help="Create json result file", + action="store_true") +args = parser.parse_args() + +with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: + functest_yaml = yaml.safe_load(f) + +dirs = functest_yaml.get('general').get('directories') +FUNCTEST_REPO = dirs.get('dir_repo_functest') +COPPER_REPO = dirs.get('dir_repo_copper') + +logger = ft_logger.Logger("copper").getLogger() + + +def main(): + cmd = "%s/tests/run.sh %s/tests" % (COPPER_REPO, COPPER_REPO) + + start_time = time.time() + + ret_val = functest_utils.execute_command(cmd, logger, exit_on_error=False) + + stop_time = time.time() + duration = round(stop_time - start_time, 1) + if ret_val == 0: + logger.info("COPPER PASSED") + test_status = 'PASS' + else: + logger.info("COPPER FAILED") + test_status = 'FAIL' + + details = { + 'timestart': start_time, + 'duration': duration, + 'status': test_status, + } + functest_utils.logger_test_results(logger, "Copper", + "copper-notification", + details['status'], details) + try: + if args.report: + functest_utils.push_results_to_db("copper", + "copper-notification", + logger, + start_time, + stop_time, + details['status'], + details) + logger.info("COPPER results pushed to DB") + except: + logger.error("Error pushing results into Database '%s'" + % sys.exc_info()[0]) + + if ret_val != 0: + sys.exit(-1) + + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/testcases/features/doctor.py b/testcases/features/doctor.py index 184ab032b..154cfc695 100755 --- a/testcases/features/doctor.py +++ b/testcases/features/doctor.py @@ -13,7 +13,7 @@ # 0.2: measure test duration and publish results under json format # # - +import argparse import os import time import yaml @@ -21,13 +21,18 @@ import yaml import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as functest_utils +parser = argparse.ArgumentParser() +parser.add_argument("-r", "--report", + help="Create json result file", + action="store_true") +args = parser.parse_args() + with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: functest_yaml = yaml.safe_load(f) dirs = functest_yaml.get('general').get('directories') FUNCTEST_REPO = dirs.get('dir_repo_functest') DOCTOR_REPO = dirs.get('dir_repo_doctor') -TEST_DB_URL = functest_yaml.get('results').get('test_db_url') logger = ft_logger.Logger("doctor").getLogger() @@ -55,32 +60,21 @@ def main(): 'duration': duration, 'status': test_status, } - pod_name = functest_utils.get_pod_name(logger) - scenario = functest_utils.get_scenario(logger) - version = functest_utils.get_version(logger) - build_tag = functest_utils.get_build_tag(logger) - status = "FAIL" if details['status'] == "OK": status = "PASS" - - logger.info("Pushing Doctor results: TEST_DB_URL=%(db)s pod_name=%(pod)s " - "version=%(v)s scenario=%(s)s criteria=%(c)s details=%(d)s" % { - 'db': TEST_DB_URL, - 'pod': pod_name, - 'v': version, - 's': scenario, - 'c': status, - 'b': build_tag, - 'd': details, - }) - functest_utils.push_results_to_db("doctor", - "doctor-notification", - logger, - start_time, - stop_time, - status, - details) + functest_utils.logger_test_results(logger, "Doctor", + "doctor-notification", + status, details) + if args.report: + functest_utils.push_results_to_db("doctor", + "doctor-notification", + logger, + start_time, + stop_time, + status, + details) + logger.info("Doctor results pushed to DB") exit(exit_code) diff --git a/testcases/features/domino.py b/testcases/features/domino.py index e4229b4c7..03ad4a2af 100755 --- a/testcases/features/domino.py +++ b/testcases/features/domino.py @@ -11,9 +11,10 @@ # Later, the VM2 boots then execute cloud-init to ping VM1. # After successful ping, both the VMs are deleted. # 0.2: measure test duration and publish results under json format -# +# 0.3: add report flag to push results when needed # +import argparse import os import time import yaml @@ -21,13 +22,19 @@ import yaml import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as functest_utils +parser = argparse.ArgumentParser() + +parser.add_argument("-r", "--report", + help="Create json result file", + action="store_true") +args = parser.parse_args() + with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: functest_yaml = yaml.safe_load(f) dirs = functest_yaml.get('general').get('directories') FUNCTEST_REPO = dirs.get('dir_repo_functest') DOMINO_REPO = dirs.get('dir_repo_domino') -TEST_DB_URL = functest_yaml.get('results').get('test_db_url') logger = ft_logger.Logger("domino").getLogger() @@ -55,10 +62,6 @@ def main(): 'duration': duration, 'status': test_status, } - pod_name = functest_utils.get_pod_name(logger) - scenario = functest_utils.get_scenario(logger) - version = functest_utils.get_version(logger) - build_tag = functest_utils.get_build_tag(logger) status = "FAIL" if details['status'] == "OK": @@ -66,25 +69,20 @@ def main(): elif details['status'] == "SKIPPED": status = "SKIP" - logger.info("Pushing Domino results: TEST_DB_URL=%(db)s pod_name=%(pod)s " - "version=%(v)s scenario=%(s)s criteria=%(c)s details=%(d)s" % { - 'db': TEST_DB_URL, - 'pod': pod_name, - 'v': version, - 's': scenario, - 'c': status, - 'b': build_tag, - 'd': details, - }) - - if status is not "SKIP": - functest_utils.push_results_to_db("domino", - "domino-multinode", - logger, - start_time, - stop_time, - status, - details) + functest_utils.logger_test_results(logger, "Domino", + "domino-multinode", + status, details) + if args.report: + if status is not "SKIP": + functest_utils.push_results_to_db("domino", + "domino-multinode", + logger, + start_time, + stop_time, + status, + details) + logger.info("Domino results pushed to DB") + if __name__ == '__main__': main() diff --git a/testcases/features/sfc/set-up-tacker.sh b/testcases/features/sfc/set-up-tacker.sh index 577b7d94e..da2cb922e 100755 --- a/testcases/features/sfc/set-up-tacker.sh +++ b/testcases/features/sfc/set-up-tacker.sh @@ -1,8 +1,8 @@ -curl "https://gerrit.opnfv.org/gerrit/gitweb?p=fuel.git;a=blob_plain;f=prototypes/sfc_tacker/poc.tacker-up.sh;hb=ddd4e11bb8bc62b7e8b06d4b44a308293c2c3362" > poc.tacker-up.sh +git_commit=ee3046f24df0bfca7ee15501f6c06ad86dd462c2 +curl "https://gerrit.opnfv.org/gerrit/gitweb?p=fuel.git;a=blob_plain;\ +f=prototypes/sfc_tacker/poc.tacker-up.sh;hb=${git_commit}" > poc.tacker-up.sh bash poc.tacker-up.sh -touch delete.sh - cat <<EOF > delete.sh tacker sfc-classifier-delete red_http tacker sfc-classifier-delete blue_ssh diff --git a/testcases/features/sfc/sfc.py b/testcases/features/sfc/sfc.py index 17ab386f6..f8af51060 100755 --- a/testcases/features/sfc/sfc.py +++ b/testcases/features/sfc/sfc.py @@ -349,6 +349,9 @@ def main(): TACKER_CHANGECLASSI subprocess.call(tacker_classi, shell=True) + logger.info("Wait for ODL to update the classification rules in OVS") + time.sleep(10) + # SSH to modify the classification flows in compute contr_cmd4 = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2" @@ -405,6 +408,21 @@ def main(): logger.info('\033[92m' + "SFC TEST WORKED" " :) \n" + '\033[0m') + # TODO report results to DB + # functest_utils.logger_test_results(logger, "SFC", + # "odl-sfc", + # status, details) + # see doctor, promise, domino, ... + # if args.report: + # logger.info("Pushing odl-SFC results") + # functest_utils.push_results_to_db("functest", + # "odl-sfc", + # logger, + # start_time, + # stop_time, + # status, + # details) + sys.exit(0) if __name__ == '__main__': diff --git a/testcases/security_scan/scripts/internet_check.py b/testcases/security_scan/scripts/internet_check.py new file mode 100644 index 000000000..1bed50a70 --- /dev/null +++ b/testcases/security_scan/scripts/internet_check.py @@ -0,0 +1,25 @@ +#!/usr/bin/python +# +# Copyright (c) 2016 Red Hat +# Luke Hinds (lhinds@redhat.com) +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Performs simple connection check, falls to default timeout of 10 seconds + +import socket + +TEST_HOST = "google.com" + + +def is_connected(): + try: + host = socket.gethostbyname(TEST_HOST) + socket.create_connection((host, 80), 2) + return True + except: + return False +print is_connected() diff --git a/testcases/security_scan/security_scan.py b/testcases/security_scan/security_scan.py index d39c29052..36b795431 100755 --- a/testcases/security_scan/security_scan.py +++ b/testcases/security_scan/security_scan.py @@ -73,21 +73,27 @@ def run_tests(host, nodetype): port = cfgparse.get(nodetype, 'port') connect.logger.info("Host: {0} Selected Profile: {1}".format(host, nodetype)) - connect.logger.info("Creating temp file structure..") - createfiles(host, port, user, localkey) - connect.logger.info("Installing OpenSCAP...") - install_pkg(host, port, user, localkey) - connect.logger.info("Running scan...") - run_scanner(host, port, user, localkey, nodetype) - clean = cfgparse.get(nodetype, 'clean') - connect.logger.info("Post installation tasks....") - post_tasks(host, port, user, localkey, nodetype) - if clean: - connect.logger.info("Cleaning down environment....") - connect.logger.info("Removing OpenSCAP....") - removepkg(host, port, user, localkey, nodetype) - connect.logger.info("Deleting tmp file and reports (remote)...") - cleandir(host, port, user, localkey, nodetype) + connect.logger.info("Checking internet for package installation...") + if internet_check(host, nodetype): + connect.logger.info("Internet Connection OK.") + connect.logger.info("Creating temp file structure..") + createfiles(host, port, user, localkey) + connect.logger.info("Installing OpenSCAP...") + install_pkg(host, port, user, localkey) + connect.logger.info("Running scan...") + run_scanner(host, port, user, localkey, nodetype) + clean = cfgparse.get(nodetype, 'clean') + connect.logger.info("Post installation tasks....") + post_tasks(host, port, user, localkey, nodetype) + if clean: + connect.logger.info("Cleaning down environment....") + connect.logger.info("Removing OpenSCAP....") + removepkg(host, port, user, localkey, nodetype) + connect.logger.info("Deleting tmp file and reports (remote)...") + cleandir(host, port, user, localkey, nodetype) + else: + connect.logger.error("Internet timeout. Moving on to next node..") + pass def nova_iterate(): @@ -106,6 +112,22 @@ def nova_iterate(): run_tests(host, nodetype) +def internet_check(host, nodetype): + import connect + user = cfgparse.get(nodetype, 'user') + port = cfgparse.get(nodetype, 'port') + localpath = functest_dir + 'scripts/internet_check.py' + remotepath = '/tmp/internet_check.py' + com = 'python /tmp/internet_check.py' + testconnect = connect.ConnectionManager(host, port, user, localkey, + localpath, remotepath, com) + connectionresult = testconnect.remotescript() + if connectionresult.rstrip() == 'True': + return True + else: + return False + + def createfiles(host, port, user, localkey): import connect global tmpdir diff --git a/testcases/vnf/vRNC/parser.py b/testcases/vnf/vRNC/parser.py index 9678591f6..40d28c49f 100644 --- a/testcases/vnf/vRNC/parser.py +++ b/testcases/vnf/vRNC/parser.py @@ -44,7 +44,10 @@ def main(): cmd = 'cd %s/tests && ./functest_run.sh' % PARSER_REPO start_time = time.time() - ret = functest_utils.execute_command(cmd, logger, exit_on_error=False) + ret = functest_utils.execute_command(cmd, + logger, + info=True, + exit_on_error=False) stop_time = time.time() status, details = functest_utils.check_test_result(project, |