diff options
| -rw-r--r-- | docker/core/Dockerfile | 6 | ||||
| -rw-r--r-- | docker/core/Switch-to-threading.Thread-for-Rally-tasks.patch | 73 | ||||
| -rw-r--r-- | docker/core/Try-to-detect-the-race-conditions.patch | 107 |
3 files changed, 76 insertions, 110 deletions
diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile index e5481d0ac..5c5008b3d 100644 --- a/docker/core/Dockerfile +++ b/docker/core/Dockerfile @@ -3,7 +3,7 @@ FROM alpine:3.11 ARG BRANCH=master ARG OPENSTACK_TAG=master -COPY Try-to-detect-the-race-conditions.patch /tmp/Try-to-detect-the-race-conditions.patch +COPY Switch-to-threading.Thread-for-Rally-tasks.patch /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch RUN apk --no-cache add --update \ python3 libffi openssl libjpeg-turbo py3-pip bash \ grep sed wget ca-certificates git openssh-client qemu-img iputils coreutils mailcap libstdc++ && \ @@ -24,9 +24,9 @@ RUN apk --no-cache add --update \ update-requirements -s --source /src/openstack-requirements /src/functest && \ pip3 install --no-cache-dir --src /src -cupper-constraints.opnfv.txt -cupper-constraints.txt \ /src/functest && \ - (cd /src/rally && patch -p1 < /tmp/Try-to-detect-the-race-conditions.patch) && \ + (cd /src/rally && patch -p1 < /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch) && \ rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/functest \ - /tmp/Try-to-detect-the-race-conditions.patch && \ + /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch && \ cp /usr/lib/python3.8/site-packages/functest/ci/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \ cp /usr/lib/python3.8/site-packages/functest/ci/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \ bash -c "mkdir -p /var/lib/xtesting /home/opnfv" && \ diff --git a/docker/core/Switch-to-threading.Thread-for-Rally-tasks.patch b/docker/core/Switch-to-threading.Thread-for-Rally-tasks.patch new file mode 100644 index 000000000..d65967ad6 --- /dev/null +++ b/docker/core/Switch-to-threading.Thread-for-Rally-tasks.patch @@ -0,0 +1,73 @@ +From 9b07423c246e7e4ab9fa25851d79ce6986c10c2e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?C=C3=A9dric=20Ollivier?= <cedric.ollivier@orange.com> +Date: Wed, 3 Jun 2020 15:23:59 +0200 +Subject: [PATCH] Switch to threading.Thread() for Rally tasks +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +multiprocessing.Process() often fails due to thread crashes [1]. +It looks similar to gsutil release notes [2]. + +[1] https://build.opnfv.org/ci/job/functest-opnfv-functest-benchmarking-cntt-latest-rally_full_cntt-run/35/console +[2] https://github.com/GoogleCloudPlatform/gsutil/issues/548 +[3] https://github.com/GoogleCloudPlatform/gsutil/blob/master/CHANGES.md + +Change-Id: I582933832e23d188c7fa5999e713dd5d7e82d2da +Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com> +--- + rally/cli/main.py | 5 ++++- + rally/task/runner.py | 7 ++++--- + 2 files changed, 8 insertions(+), 4 deletions(-) + +diff --git a/rally/cli/main.py b/rally/cli/main.py +index 235a57113..14c057c0e 100644 +--- a/rally/cli/main.py ++++ b/rally/cli/main.py +@@ -15,6 +15,10 @@ + + """CLI interface for Rally.""" + ++STACK_SIZE = 1024 * 1024 ++import threading ++threading.stack_size(STACK_SIZE) ++ + import sys + + from rally.cli import cliutils +@@ -25,7 +29,6 @@ from rally.cli.commands import plugin + from rally.cli.commands import task + from rally.cli.commands import verify + +- + categories = { + "db": db.DBCommands, + "env": env.EnvCommands, +diff --git a/rally/task/runner.py b/rally/task/runner.py +index 3397e1193..5edebb406 100644 +--- a/rally/task/runner.py ++++ b/rally/task/runner.py +@@ -17,6 +17,7 @@ import abc + import collections + import copy + import multiprocessing ++import threading + import time + + from rally.common import logging +@@ -186,9 +187,9 @@ class ScenarioRunner(plugin.Plugin, validation.ValidatablePluginMixin, + for i in range(processes_to_start): + kwrgs = {"processes_to_start": processes_to_start, + "processes_counter": i} +- process = multiprocessing.Process(target=worker_process, +- args=next(worker_args_gen), +- kwargs={"info": kwrgs}) ++ process = threading.Thread(target=worker_process, ++ args=next(worker_args_gen), ++ kwargs={"info": kwrgs}) + process.start() + process_pool.append(process) + +-- +2.26.2 + diff --git a/docker/core/Try-to-detect-the-race-conditions.patch b/docker/core/Try-to-detect-the-race-conditions.patch deleted file mode 100644 index 9413d7617..000000000 --- a/docker/core/Try-to-detect-the-race-conditions.patch +++ /dev/null @@ -1,107 +0,0 @@ -From 304497b81fbbe9cb8608b947cae76aeaa2b0934e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?C=C3=A9dric=20Ollivier?= <cedric.ollivier@orange.com> -Date: Wed, 3 Jun 2020 15:23:59 +0200 -Subject: [PATCH 11/11] Try to detect the race conditions -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Change-Id: I582933832e23d188c7fa5999e713dd5d7e82d2da -Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com> ---- - rally/cli/main.py | 5 ++++- - rally/task/runner.py | 23 ++++++++++++++++++----- - 2 files changed, 22 insertions(+), 6 deletions(-) - -diff --git a/rally/cli/main.py b/rally/cli/main.py -index 235a57113..14c057c0e 100644 ---- a/rally/cli/main.py -+++ b/rally/cli/main.py -@@ -15,6 +15,10 @@ - - """CLI interface for Rally.""" - -+STACK_SIZE = 1024 * 1024 -+import threading -+threading.stack_size(STACK_SIZE) -+ - import sys - - from rally.cli import cliutils -@@ -25,7 +29,6 @@ from rally.cli.commands import plugin - from rally.cli.commands import task - from rally.cli.commands import verify - -- - categories = { - "db": db.DBCommands, - "env": env.EnvCommands, -diff --git a/rally/task/runner.py b/rally/task/runner.py -index 3397e1193..b2fde8550 100644 ---- a/rally/task/runner.py -+++ b/rally/task/runner.py -@@ -17,6 +17,7 @@ import abc - import collections - import copy - import multiprocessing -+import threading - import time - - from rally.common import logging -@@ -51,10 +52,14 @@ def _get_scenario_context(iteration, context_obj): - def _run_scenario_once(cls, method_name, context_obj, scenario_kwargs, - event_queue): - iteration = context_obj["iteration"] -+ LOG.info("DEBUGRACE %s putting in event_queue iteration: %s", -+ threading.get_native_id(), iteration) - event_queue.put({ - "type": "iteration", - "value": iteration, - }) -+ LOG.info("DEBUGRACE %s put in event_queue iteration: %s", -+ threading.get_native_id(), iteration) - - # provide arguments isolation between iterations - scenario_kwargs = copy.deepcopy(scenario_kwargs) -@@ -65,6 +70,8 @@ def _run_scenario_once(cls, method_name, context_obj, scenario_kwargs, - scenario_inst = cls(context_obj) - error = [] - try: -+ LOG.info("DEBUGRACE %s running %s %s", -+ threading.get_native_id(), scenario_inst, scenario_inst) - with rutils.Timer() as timer: - getattr(scenario_inst, method_name)(**scenario_kwargs) - except Exception as e: -@@ -87,8 +94,14 @@ def _run_scenario_once(cls, method_name, context_obj, scenario_kwargs, - - def _worker_thread(queue, cls, method_name, context_obj, scenario_kwargs, - event_queue): -- queue.put(_run_scenario_once(cls, method_name, context_obj, -- scenario_kwargs, event_queue)) -+ result = _run_scenario_once(cls, method_name, context_obj, -+ scenario_kwargs, event_queue) -+ LOG.info("DEBUGRACE %s putting in result_queue context_obj: %s", -+ threading.get_native_id(), -+ context_obj) -+ queue.put(result) -+ LOG.info("DEBUGRACE %s put in result_queue context_obj: %s: %s", -+ threading.get_native_id(), context_obj, result) - - - def _log_worker_info(**info): -@@ -186,9 +199,9 @@ class ScenarioRunner(plugin.Plugin, validation.ValidatablePluginMixin, - for i in range(processes_to_start): - kwrgs = {"processes_to_start": processes_to_start, - "processes_counter": i} -- process = multiprocessing.Process(target=worker_process, -- args=next(worker_args_gen), -- kwargs={"info": kwrgs}) -+ process = threading.Thread(target=worker_process, -+ args=next(worker_args_gen), -+ kwargs={"info": kwrgs}) - process.start() - process_pool.append(process) - --- -2.26.2 - |