From 10ce40881184217bd72ec3e6cae474fe77a2e6c9 Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Thu, 22 Aug 2019 16:35:10 +0200 Subject: Protect vs rally infine loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It calls timeout (1 hour) to stop any hanging task [1]. It will be backported to Hunter and Iruya if similar failures are detected. It occurs vs Iruya too [2]. [1] https://build.opnfv.org/ci/job/functest-opnfv-functest-benchmarking-latest-rally_full-run/9/console [2] https://build.opnfv.org/ci/job/functest-opnfv-functest-benchmarking-iruya-rally_jobs-run/151/console Change-Id: I8fe89f9fa1c6c39cd8810096c1d586bdef7865a0 Signed-off-by: Cédric Ollivier (cherry picked from commit fc5acf5d25f61487947df08477b9f239f40e486d) --- functest/opnfv_tests/openstack/rally/rally.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 22b508afa..f6e563ba7 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -67,6 +67,7 @@ class RallyBase(singlevm.VmReady2): visibility = 'public' shared_network = True allow_no_fip = True + task_timeout = '3600' def __init__(self, **kwargs): """Initialize RallyBase object.""" @@ -524,7 +525,8 @@ class RallyBase(singlevm.VmReady2): if self.file_is_empty(file_name): LOGGER.info('No tests for scenario "%s"', test_name) return False - self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure", + self.run_cmd = (["timeout", "-t", self.task_timeout, + "rally", "task", "start", "--abort-on-sla-failure", "--task", self.task_file, "--task-args", str(self.build_task_args(test_name))]) return True @@ -729,6 +731,7 @@ class RallyJobs(RallyBase): """Rally OpenStack CI testcase implementation.""" stests = ["neutron"] + task_timeout = '7200' def __init__(self, **kwargs): """Initialize RallyJobs object.""" @@ -834,7 +837,8 @@ class RallyJobs(RallyBase): os.makedirs(self.temp_dir) task_file_name = os.path.join(self.temp_dir, task_name) self.apply_blacklist(task, task_file_name) - self.run_cmd = (["rally", "task", "start", "--task", task_file_name, + self.run_cmd = (["timeout", "-t", self.task_timeout, + "rally", "task", "start", "--task", task_file_name, "--task-args", str(self.build_task_args(test_name))]) return True -- cgit 1.2.3-korg