diff options
| author |   Juha Kosonen <juha.kosonen@nokia.com> | 2019-09-04 15:21:00 +0300 |
|---|---|---|
| committer | Juha Kosonen <juha.kosonen@nokia.com> | 2019-09-04 15:21:00 +0300 |
| commit | b751a36eb2c6269aa44d314b3f5797db17742e6a (patch) | |
| tree | 86b8cde2e4e6f5eb475a724812dc8150920daf06 | |
| parent | 4ef1169759eaeb39144ac7e2a3f814ea86af7a67 (diff) | |
Prevent occasional rally hangs
Set timeout on subprocess invocation instead of spawned child process.
Increase the timeout value for rally_full to 2h since the execution
typically takes ~90min [1].
[1] https://build.opnfv.org/ci/job/functest-opnfv-functest-benchmarking-latest-rally_full-run/22/
Change-Id: I0ca90bc2d85b4625336eb0396d8b2816a486b746
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 21 |
1 files changed, 13 insertions, 8 deletions
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 46c1d95da..9a04f3873 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -67,7 +67,7 @@ class RallyBase(singlevm.VmReady2): visibility = 'public' shared_network = True allow_no_fip = True - task_timeout = '3600' + task_timeout = 3600 def __init__(self, **kwargs): """Initialize RallyBase object.""" @@ -429,8 +429,13 @@ class RallyBase(singlevm.VmReady2): LOGGER.debug('running command: %s', self.run_cmd) proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = proc.communicate()[0] - + try: + output = proc.communicate(timeout=self.task_timeout)[0] + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + LOGGER.error("Failed to complete run task") + raise Exception("Failed to complete run task") task_id = self.get_task_id(output) LOGGER.debug('task_id : %s', task_id) if task_id is None: @@ -525,8 +530,7 @@ class RallyBase(singlevm.VmReady2): if self.file_is_empty(file_name): LOGGER.info('No tests for scenario "%s"', test_name) return False - self.run_cmd = (["timeout", self.task_timeout, - "rally", "task", "start", "--abort-on-sla-failure", + self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure", "--task", self.task_file, "--task-args", str(self.build_task_args(test_name))]) return True @@ -718,6 +722,8 @@ class RallySanity(RallyBase): class RallyFull(RallyBase): """Rally full testcase implementation.""" + task_timeout = 7200 + def __init__(self, **kwargs): """Initialize RallyFull object.""" if "case_name" not in kwargs: @@ -731,7 +737,7 @@ class RallyJobs(RallyBase): """Rally OpenStack CI testcase implementation.""" stests = ["neutron"] - task_timeout = '7200' + task_timeout = 7200 def __init__(self, **kwargs): """Initialize RallyJobs object.""" @@ -837,8 +843,7 @@ class RallyJobs(RallyBase): os.makedirs(self.temp_dir) task_file_name = os.path.join(self.temp_dir, task_name) self.apply_blacklist(task, task_file_name) - self.run_cmd = (["timeout", self.task_timeout, - "rally", "task", "start", "--task", task_file_name, + self.run_cmd = (["rally", "task", "start", "--task", task_file_name, "--task-args", str(self.build_task_args(test_name))]) return True |