diff options
Diffstat (limited to 'docker/tempest')
6 files changed, 598 insertions, 7 deletions
diff --git a/docker/tempest/Accept-custom-registered-endpoints.patch b/docker/tempest/Accept-custom-registered-endpoints.patch new file mode 100644 index 000000000..eba6ff436 --- /dev/null +++ b/docker/tempest/Accept-custom-registered-endpoints.patch @@ -0,0 +1,99 @@ +From 1d500e79156ada6bc6fdb628ed1da0efd4121f6a Mon Sep 17 00:00:00 2001 +From: Martin Kopec <mkopec@redhat.com> +Date: Thu, 31 Oct 2019 13:56:42 +0000 +Subject: [PATCH 13/46] Accept custom registered endpoints + +The review drops usage of skip_path() filter in the related tests +and uses raw_request() instead. + +Normally a swift url is organised as host:port/info and +host:port/v1/AUTH_<tenant-id>, see +https://docs.openstack.org/api-ref/object-store/ +But RadosGW API is organised as host:port/swift/info and +host:port/swift/v1/AUTH_<tenant-id>, see +https://docs.ceph.com/docs/master/radosgw/config-ref/#swift-settings + +Close-bug: 1799981 +Change-Id: I6a932639a05defe0f04c600afcc35a19662937af +--- + tempest/api/object_storage/test_crossdomain.py | 7 +++---- + tempest/api/object_storage/test_healthcheck.py | 7 +++---- + tempest/lib/services/object_storage/capabilities_client.py | 5 +++-- + .../services/object_storage/test_capabilities_client.py | 2 +- + 4 files changed, 10 insertions(+), 11 deletions(-) + +diff --git a/tempest/api/object_storage/test_crossdomain.py b/tempest/api/object_storage/test_crossdomain.py +index f61d9f891..1567e0640 100644 +--- a/tempest/api/object_storage/test_crossdomain.py ++++ b/tempest/api/object_storage/test_crossdomain.py +@@ -34,13 +34,12 @@ class CrossdomainTest(base.BaseObjectTest): + def setUp(self): + super(CrossdomainTest, self).setUp() + +- # Turning http://.../v1/foobar into http://.../ +- self.account_client.skip_path() +- + @decorators.idempotent_id('d1b8b031-b622-4010-82f9-ff78a9e915c7') + @utils.requires_ext(extension='crossdomain', service='object') + def test_get_crossdomain_policy(self): +- resp, body = self.account_client.get("crossdomain.xml", {}) ++ url = self.account_client._get_base_version_url() + "crossdomain.xml" ++ resp, body = self.account_client.raw_request(url, "GET") ++ self.account_client._error_checker(resp, body) + body = body.decode() + + self.assertTrue(body.startswith(self.xml_start) and +diff --git a/tempest/api/object_storage/test_healthcheck.py b/tempest/api/object_storage/test_healthcheck.py +index a186f9ee6..8e9e4061d 100644 +--- a/tempest/api/object_storage/test_healthcheck.py ++++ b/tempest/api/object_storage/test_healthcheck.py +@@ -22,13 +22,12 @@ class HealthcheckTest(base.BaseObjectTest): + + def setUp(self): + super(HealthcheckTest, self).setUp() +- # Turning http://.../v1/foobar into http://.../ +- self.account_client.skip_path() + + @decorators.idempotent_id('db5723b1-f25c-49a9-bfeb-7b5640caf337') + def test_get_healthcheck(self): +- +- resp, _ = self.account_client.get("healthcheck", {}) ++ url = self.account_client._get_base_version_url() + "healthcheck" ++ resp, body = self.account_client.raw_request(url, "GET") ++ self.account_client._error_checker(resp, body) + + # The target of the request is not any Swift resource. Therefore, the + # existence of response header is checked without a custom matcher. +diff --git a/tempest/lib/services/object_storage/capabilities_client.py b/tempest/lib/services/object_storage/capabilities_client.py +index d31bbc299..f08bd9aea 100644 +--- a/tempest/lib/services/object_storage/capabilities_client.py ++++ b/tempest/lib/services/object_storage/capabilities_client.py +@@ -21,9 +21,10 @@ from tempest.lib.common import rest_client + class CapabilitiesClient(rest_client.RestClient): + + def list_capabilities(self): +- self.skip_path() + try: +- resp, body = self.get('info') ++ url = self._get_base_version_url() + 'info' ++ resp, body = self.raw_request(url, 'GET') ++ self._error_checker(resp, body) + finally: + self.reset_path() + body = json.loads(body) +diff --git a/tempest/tests/lib/services/object_storage/test_capabilities_client.py b/tempest/tests/lib/services/object_storage/test_capabilities_client.py +index b7f972a85..9df7c7c18 100644 +--- a/tempest/tests/lib/services/object_storage/test_capabilities_client.py ++++ b/tempest/tests/lib/services/object_storage/test_capabilities_client.py +@@ -43,7 +43,7 @@ class TestCapabilitiesClient(base.BaseServiceTest): + } + self.check_service_client_function( + self.client.list_capabilities, +- 'tempest.lib.common.rest_client.RestClient.get', ++ 'tempest.lib.common.rest_client.RestClient.raw_request', + resp, + bytes_body) + +-- +2.26.0.rc2 + diff --git a/docker/tempest/Create-new-server-in-test_create_backup.patch b/docker/tempest/Create-new-server-in-test_create_backup.patch new file mode 100644 index 000000000..1b86b0fc5 --- /dev/null +++ b/docker/tempest/Create-new-server-in-test_create_backup.patch @@ -0,0 +1,84 @@ +From 03eb38ce54aeec4bc4c1cb3475c6fb84661f8993 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?C=C3=A9dric=20Ollivier?= <cedric.ollivier@orange.com> +Date: Tue, 21 Jul 2020 13:28:50 +0200 +Subject: [PATCH] Create new server in test_create_backup +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +test_reboot_server_hard sometimes fail in all gates [1]. +This hack could highlight if they are side effects between +test_create_backup and test_reboot_server_hard. + +[1] http://artifacts.opnfv.org/functest/E5AZMH89OOK6/functest-opnfv-functest-smoke-cntt-hunter-tempest_full_cntt-run-142/tempest_full_cntt/tempest-report.html + +Change-Id: I203562f686b004094e5e18858004b7a2d26567a6 +Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com> +--- + .../api/compute/servers/test_server_actions.py | 15 ++++++++------- + 1 file changed, 8 insertions(+), 7 deletions(-) + +diff --git a/tempest/api/compute/servers/test_server_actions.py b/tempest/api/compute/servers/test_server_actions.py +index d477be0eb..c369311d3 100644 +--- a/tempest/api/compute/servers/test_server_actions.py ++++ b/tempest/api/compute/servers/test_server_actions.py +@@ -443,6 +443,7 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest): + # Check if glance v1 is available to determine which client to use. We + # prefer glance v1 for the compute API tests since the compute image + # API proxy was written for glance v1. ++ newserver = self.create_test_server(wait_until='ACTIVE') + if CONF.image_feature_enabled.api_v1: + glance_client = self.os_primary.image_client + elif CONF.image_feature_enabled.api_v2: +@@ -453,7 +454,7 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest): + '[image-feature-enabled].') + + backup1 = data_utils.rand_name('backup-1') +- resp = self.client.create_backup(self.server_id, ++ resp = self.client.create_backup(newserver['id'], + backup_type='daily', + rotation=2, + name=backup1) +@@ -481,8 +482,8 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest): + image1_id, 'active') + + backup2 = data_utils.rand_name('backup-2') +- waiters.wait_for_server_status(self.client, self.server_id, 'ACTIVE') +- resp = self.client.create_backup(self.server_id, ++ waiters.wait_for_server_status(self.client, newserver['id'], 'ACTIVE') ++ resp = self.client.create_backup(newserver['id'], + backup_type='daily', + rotation=2, + name=backup2) +@@ -499,7 +500,7 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest): + properties = { + 'image_type': 'backup', + 'backup_type': "daily", +- 'instance_uuid': self.server_id, ++ 'instance_uuid': newserver['id'], + } + params = { + 'status': 'active', +@@ -524,8 +525,8 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest): + # create the third one, due to the rotation is 2, + # the first one will be deleted + backup3 = data_utils.rand_name('backup-3') +- waiters.wait_for_server_status(self.client, self.server_id, 'ACTIVE') +- resp = self.client.create_backup(self.server_id, ++ waiters.wait_for_server_status(self.client, newserver['id'], 'ACTIVE') ++ resp = self.client.create_backup(newserver['id'], + backup_type='daily', + rotation=2, + name=backup3) +@@ -536,7 +537,7 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest): + image3_id = data_utils.parse_image_id(resp.response['location']) + self.addCleanup(glance_client.delete_image, image3_id) + # the first back up should be deleted +- waiters.wait_for_server_status(self.client, self.server_id, 'ACTIVE') ++ waiters.wait_for_server_status(self.client, newserver['id'], 'ACTIVE') + glance_client.wait_for_resource_deletion(image1_id) + oldest_backup_exist = False + if CONF.image_feature_enabled.api_v1: +-- +2.27.0 + diff --git a/docker/tempest/Dockerfile b/docker/tempest/Dockerfile index 2835285f6..915bcb6f5 100644 --- a/docker/tempest/Dockerfile +++ b/docker/tempest/Dockerfile @@ -5,31 +5,53 @@ ARG OPENSTACK_TAG=stable/stein ARG TEMPEST_TAG=21.0.0 ARG RALLY_TAG=1.5.1 ARG RALLY_OPENSTACK_TAG=1.5.0 -ARG UJSON_TAG=d25e024f481c5571d15f3c0c406a498ca0467cfd +COPY Accept-custom-registered-endpoints.patch /tmp/Accept-custom-registered-endpoints.patch +COPY Fixes-race-condition-in-test_add_remove_fixed_ip.patch /tmp/Fixes-race-condition-in-test_add_remove_fixed_ip.patch +COPY object-storage-fix-and-cleanup-header-checks.patch /tmp/object-storage-fix-and-cleanup-header-checks.patch +COPY Create-new-server-in-test_create_backup.patch /tmp/Create-new-server-in-test_create_backup.patch +COPY Switch-to-threading.Thread-for-Rally-tasks.patch /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch RUN apk --no-cache add --virtual .build-deps --update \ - python-dev build-base linux-headers libffi-dev \ + python3-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/opendev.org\\/openstack\\/tempest@$TEMPEST_TAG#egg=tempest/ upper-constraints.txt && \ - sed -i -E s/^ujson==+.*$/-e\ git+https:\\/\\/github.com\\/esnme\\/ultrajson@$UJSON_TAG#egg=ujson/ upper-constraints.txt && \ + sed -i -E /^ujson==+.*$/d upper-constraints.txt && \ + sed -i -E /^kubernetes==+.*$/d upper-constraints.txt && \ case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ + sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \ + sed -i -E /#egg=xrally-kubernetes/d upper-constraints.opnfv.txt && \ git init /src/rally && \ (cd /src/rally && \ git fetch --tags https://opendev.org/openstack/rally.git $RALLY_TAG && \ git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/rally/ && \ + (cd /src/rally && patch -p1 < /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch) && \ + pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ + /src/rally && \ (git clone https://opendev.org/openstack/rally-openstack.git /src/rally-openstack && \ cd /src/rally-openstack && git checkout $RALLY_OPENSTACK_TAG) && \ update-requirements -s --source /src/openstack-requirements /src/rally-openstack && \ - pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ + pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ tempest /src/rally-openstack && \ - pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ - /src/rally && \ - rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/rally /src/rally-openstack && \ + rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/rally /src/rally-openstack \ + /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch && \ mkdir -p /etc/rally && \ printf "[database]\nconnection = 'sqlite:////var/lib/rally/database/rally.sqlite'" > /etc/rally/rally.conf && \ mkdir -p /var/lib/rally/database && rally db create && \ + (cd /src/tempest && \ + git config --global user.email "opnfv-tech-discuss@lists.opnfv.org" && \ + git config --global user.name "Functest" && \ + patch -p1 < /tmp/Accept-custom-registered-endpoints.patch && \ + patch -p1 < /tmp/object-storage-fix-and-cleanup-header-checks.patch && \ + patch -p1 < /tmp/Fixes-race-condition-in-test_add_remove_fixed_ip.patch && \ + patch -p1 < /tmp/Create-new-server-in-test_create_backup.patch && \ + git commit -a -m "Backport critical bugfixes" && \ + rm ~/.gitconfig) && \ + rm /tmp/Accept-custom-registered-endpoints.patch \ + /tmp/object-storage-fix-and-cleanup-header-checks.patch \ + /tmp/Fixes-race-condition-in-test_add_remove_fixed_ip.patch \ + /tmp/Create-new-server-in-test_create_backup.patch && \ apk del .build-deps diff --git a/docker/tempest/Fixes-race-condition-in-test_add_remove_fixed_ip.patch b/docker/tempest/Fixes-race-condition-in-test_add_remove_fixed_ip.patch new file mode 100644 index 000000000..a9f3c7e6d --- /dev/null +++ b/docker/tempest/Fixes-race-condition-in-test_add_remove_fixed_ip.patch @@ -0,0 +1,165 @@ +From 61a3c8efa4a5e41dc6b5fd2d7a28a25555ebb54b Mon Sep 17 00:00:00 2001 +From: David Sedlák <dsedlak@redhat.com> +Date: Wed, 30 Oct 2019 15:38:21 +0100 +Subject: [PATCH] Fixes race condition in test_add_remove_fixed_ip + +Currently race condition can occure in +tempest.api.compute.servers.test_attach_interfaces. +AttachInterfacesUnderV243Test.test_add_remove_fixed_ip +when floating IP added during resource preparation doesn't appear in +the list of original IPs that is created at the beggining of the test, +which then confuses the test +and floating ip is later recognized as fixed IP added in the test. +more details including log: +https://bugzilla.redhat.com/show_bug.cgi?id=1752416 + +This change ensures possible floating IP added during server +creation is always present in the set of original IPs and also +during every comparasion of IPs. + +Closes-Bug: #1866179 + +Change-Id: Ic3a3e0708714b6d6c9c226e641e1c520e5ebde9d +Signed-off-by: David Sedlák <dsedlak@redhat.com> +--- + +diff --git a/tempest/api/compute/servers/test_attach_interfaces.py b/tempest/api/compute/servers/test_attach_interfaces.py +index df8da07..c1af6c7 100644 +--- a/tempest/api/compute/servers/test_attach_interfaces.py ++++ b/tempest/api/compute/servers/test_attach_interfaces.py +@@ -86,12 +86,16 @@ + # apparently not enough? Add cleanup here. + self.addCleanup(self.delete_server, server['id']) + self._wait_for_validation(server, validation_resources) ++ try: ++ fip = set([validation_resources['floating_ip']['ip']]) ++ except KeyError: ++ fip = () + ifs = (self.interfaces_client.list_interfaces(server['id']) + ['interfaceAttachments']) + body = waiters.wait_for_interface_status( + self.interfaces_client, server['id'], ifs[0]['port_id'], 'ACTIVE') + ifs[0]['port_state'] = body['port_state'] +- return server, ifs ++ return server, ifs, fip + + + class AttachInterfacesTestJSON(AttachInterfacesTestBase): +@@ -226,7 +230,7 @@ + @decorators.idempotent_id('73fe8f02-590d-4bf1-b184-e9ca81065051') + @utils.services('network') + def test_create_list_show_delete_interfaces_by_network_port(self): +- server, ifs = self._create_server_get_interfaces() ++ server, ifs, _ = self._create_server_get_interfaces() + interface_count = len(ifs) + self.assertGreater(interface_count, 0) + +@@ -268,7 +272,7 @@ + raise self.skipException("Only owner network supports " + "creating interface by fixed ip.") + +- server, ifs = self._create_server_get_interfaces() ++ server, ifs, _ = self._create_server_get_interfaces() + interface_count = len(ifs) + self.assertGreater(interface_count, 0) + +@@ -354,9 +358,8 @@ + not CONF.network.shared_physical_network): + raise self.skipException("Only owner network supports " + "creating interface by fixed ip.") +- + # Add and Remove the fixed IP to server. +- server, ifs = self._create_server_get_interfaces() ++ server, ifs, fip = self._create_server_get_interfaces() + original_interface_count = len(ifs) # This is the number of ports. + self.assertGreater(original_interface_count, 0) + # Get the starting list of IPs on the server. +@@ -369,6 +372,9 @@ + self.assertEqual(1, len(addresses), addresses) # number of networks + # Keep track of the original addresses so we can know which IP is new. + original_ips = [addr['addr'] for addr in list(addresses.values())[0]] ++ # Make sure the floating IP possibly assigned during ++ # server creation is always present in the set of original ips. ++ original_ips = set(original_ips).union(fip) + original_ip_count = len(original_ips) + self.assertGreater(original_ip_count, 0, addresses) # at least 1 + network_id = ifs[0]['net_id'] +@@ -376,40 +382,22 @@ + # fixed IP on the same network (and same port since we only have one + # port). + self.servers_client.add_fixed_ip(server['id'], networkId=network_id) +- # Wait for the ips count to increase by one. + +- def _get_server_floating_ips(): +- _floating_ips = [] +- _server = self.os_primary.servers_client.show_server( +- server['id'])['server'] +- for _ip_set in _server['addresses']: +- for _ip in _server['addresses'][_ip_set]: +- if _ip['OS-EXT-IPS:type'] == 'floating': +- _floating_ips.append(_ip['addr']) +- return _floating_ips +- +- def _wait_for_ip_increase(): ++ def _wait_for_ip_change(expected_count): + _addresses = self.os_primary.servers_client.list_addresses( + server['id'])['addresses'] +- _ips = [addr['addr'] for addr in list(_addresses.values())[0]] +- LOG.debug("Wait for IP increase. All IPs still associated to " ++ _ips = set([addr['addr'] for addr in list(_addresses.values())[0]]) ++ # Make sure possible floating ip is always present in the set. ++ _ips = _ips.union(fip) ++ LOG.debug("Wait for change of IPs. All IPs still associated to " + "the server %(id)s: %(ips)s", + {'id': server['id'], 'ips': _ips}) +- if len(_ips) == original_ip_count + 1: +- return True +- elif len(_ips) == original_ip_count: +- return False +- # If not, lets remove any floating IP from the list and check again +- _fips = _get_server_floating_ips() +- _ips = [_ip for _ip in _ips if _ip not in _fips] +- LOG.debug("Wait for IP increase. Fixed IPs still associated to " +- "the server %(id)s: %(ips)s", +- {'id': server['id'], 'ips': _ips}) +- return len(_ips) == original_ip_count + 1 ++ return len(_ips) == expected_count + ++ # Wait for the ips count to increase by one. + if not test_utils.call_until_true( +- _wait_for_ip_increase, CONF.compute.build_timeout, +- CONF.compute.build_interval): ++ _wait_for_ip_change, CONF.compute.build_timeout, ++ CONF.compute.build_interval, original_ip_count + 1): + raise lib_exc.TimeoutException( + 'Timed out while waiting for IP count to increase.') + +@@ -428,26 +416,8 @@ + break + self.servers_client.remove_fixed_ip(server['id'], address=fixed_ip) + # Wait for the interface count to decrease by one. +- +- def _wait_for_ip_decrease(): +- _addresses = self.os_primary.servers_client.list_addresses( +- server['id'])['addresses'] +- _ips = [addr['addr'] for addr in list(_addresses.values())[0]] +- LOG.debug("Wait for IP decrease. All IPs still associated to " +- "the server %(id)s: %(ips)s", +- {'id': server['id'], 'ips': _ips}) +- if len(_ips) == original_ip_count: +- return True +- # If not, lets remove any floating IP from the list and check again +- _fips = _get_server_floating_ips() +- _ips = [_ip for _ip in _ips if _ip not in _fips] +- LOG.debug("Wait for IP decrease. Fixed IPs still associated to " +- "the server %(id)s: %(ips)s", +- {'id': server['id'], 'ips': _ips}) +- return len(_ips) == original_ip_count +- + if not test_utils.call_until_true( +- _wait_for_ip_decrease, CONF.compute.build_timeout, +- CONF.compute.build_interval): ++ _wait_for_ip_change, CONF.compute.build_timeout, ++ CONF.compute.build_interval, original_ip_count): + raise lib_exc.TimeoutException( + 'Timed out while waiting for IP count to decrease.') diff --git a/docker/tempest/Switch-to-threading.Thread-for-Rally-tasks.patch b/docker/tempest/Switch-to-threading.Thread-for-Rally-tasks.patch new file mode 100644 index 000000000..7c146c9ed --- /dev/null +++ b/docker/tempest/Switch-to-threading.Thread-for-Rally-tasks.patch @@ -0,0 +1,50 @@ +From 7223c6c766d2cbd47c54048426c904a27b52e069 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?C=C3=A9dric=20Ollivier?= <cedric.ollivier@orange.com> +Date: Wed, 3 Jun 2020 15:23:59 +0200 +Subject: [PATCH] Switch to threading.Thread() for Rally tasks +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +multiprocessing.Process() often fails due to thread crashes [1]. +It looks similar to gsutil release notes [2]. + +[1] https://build.opnfv.org/ci/job/functest-opnfv-functest-benchmarking-cntt-latest-rally_full_cntt-run/35/console +[2] https://github.com/GoogleCloudPlatform/gsutil/issues/548 +[3] https://github.com/GoogleCloudPlatform/gsutil/blob/master/CHANGES.md + +Change-Id: I582933832e23d188c7fa5999e713dd5d7e82d2da +Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com> +(cherry picked from commit 9b07423c246e7e4ab9fa25851d79ce6986c10c2e) +--- + rally/task/runner.py | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/rally/task/runner.py b/rally/task/runner.py +index 55372e509..0f0245588 100644 +--- a/rally/task/runner.py ++++ b/rally/task/runner.py +@@ -17,6 +17,7 @@ import abc + import collections + import copy + import multiprocessing ++import threading + import time + + import six +@@ -188,9 +189,9 @@ class ScenarioRunner(plugin.Plugin, validation.ValidatablePluginMixin): + for i in range(processes_to_start): + kwrgs = {"processes_to_start": processes_to_start, + "processes_counter": i} +- process = multiprocessing.Process(target=worker_process, +- args=next(worker_args_gen), +- kwargs={"info": kwrgs}) ++ process = threading.Thread(target=worker_process, ++ args=next(worker_args_gen), ++ kwargs={"info": kwrgs}) + process.start() + process_pool.append(process) + +-- +2.26.2 + diff --git a/docker/tempest/object-storage-fix-and-cleanup-header-checks.patch b/docker/tempest/object-storage-fix-and-cleanup-header-checks.patch new file mode 100644 index 000000000..629a98174 --- /dev/null +++ b/docker/tempest/object-storage-fix-and-cleanup-header-checks.patch @@ -0,0 +1,171 @@ +From 42e111c7d8f1cdb5d51a1c4f2ce5c64c3d3471f1 Mon Sep 17 00:00:00 2001 +From: Thomas Morin <thomas.morin@orange.com> +Date: Wed, 17 Jun 2020 18:08:49 +0200 +Subject: [PATCH] object storage: fix and cleanup header checks + +As explained in [1] it is not legitimate to require a Transfer-Encoding +header in Swift responses. That prevents running some tests +successfully in the case where Swift is behind a proxy/load-balancer +that does not use any Transfer-Encoding in its responses. + +This change hence removes the checks for the presence of a +"Transfer-Encoding" header, and replaces them by the use +of the existing check methods, after modifying the +custom_matcher checks on which these methods rely on to accept +either a Content-Length or a Transfer-Encoding header. + +Some adaptation was also required to avoid trying to process 'etag' +for DELETE requests. + +A side-effect of this change is a code simplification and +cleanup since the specific header checks in the corresponding +tests are replaced by the generic check methods. + +[1] https://bugs.launchpad.net/tempest/+bug/1819851/comments/3 + +Closes-Bug: 1819851 +Change-Id: Iaccea41640a53b564f72cee73981e2e61d3e80ae +--- + .../api/object_storage/test_account_bulk.py | 37 ++----------------- + tempest/api/object_storage/test_object_slo.py | 13 +------ + tempest/common/custom_matchers.py | 16 +++++++- + 3 files changed, 19 insertions(+), 47 deletions(-) + +diff --git a/tempest/api/object_storage/test_account_bulk.py b/tempest/api/object_storage/test_account_bulk.py +index 6599e432f..80f790f51 100644 +--- a/tempest/api/object_storage/test_account_bulk.py ++++ b/tempest/api/object_storage/test_account_bulk.py +@@ -16,7 +16,6 @@ import tarfile + import tempfile + + from tempest.api.object_storage import base +-from tempest.common import custom_matchers + from tempest.common import utils + from tempest.lib import decorators + +@@ -76,17 +75,7 @@ class BulkTest(base.BaseObjectTest): + resp = self._upload_archive(filepath) + self.containers.append(container_name) + +- # When uploading an archived file with the bulk operation, the response +- # does not contain 'content-length' header. This is the special case, +- # therefore the existence of response headers is checked without +- # custom matcher. +- self.assertIn('transfer-encoding', resp.response) +- self.assertIn('content-type', resp.response) +- self.assertIn('x-trans-id', resp.response) +- self.assertIn('date', resp.response) +- +- # Check only the format of common headers with custom matcher +- self.assertThat(resp.response, custom_matchers.AreAllWellFormatted()) ++ self.assertHeaders(resp.response, 'Account', 'PUT') + + param = {'format': 'json'} + resp, body = self.account_client.list_account_containers(param) +@@ -113,17 +102,7 @@ class BulkTest(base.BaseObjectTest): + data = '%s/%s\n%s' % (container_name, object_name, container_name) + resp = self.bulk_client.delete_bulk_data(data=data) + +- # When deleting multiple files using the bulk operation, the response +- # does not contain 'content-length' header. This is the special case, +- # therefore the existence of response headers is checked without +- # custom matcher. +- self.assertIn('transfer-encoding', resp.response) +- self.assertIn('content-type', resp.response) +- self.assertIn('x-trans-id', resp.response) +- self.assertIn('date', resp.response) +- +- # Check only the format of common headers with custom matcher +- self.assertThat(resp.response, custom_matchers.AreAllWellFormatted()) ++ self.assertHeaders(resp.response, 'Account', 'DELETE') + + # Check if uploaded contents are completely deleted + self._check_contents_deleted(container_name) +@@ -139,17 +118,7 @@ class BulkTest(base.BaseObjectTest): + + resp = self.bulk_client.delete_bulk_data_with_post(data=data) + +- # When deleting multiple files using the bulk operation, the response +- # does not contain 'content-length' header. This is the special case, +- # therefore the existence of response headers is checked without +- # custom matcher. +- self.assertIn('transfer-encoding', resp.response) +- self.assertIn('content-type', resp.response) +- self.assertIn('x-trans-id', resp.response) +- self.assertIn('date', resp.response) +- +- # Check only the format of common headers with custom matcher +- self.assertThat(resp.response, custom_matchers.AreAllWellFormatted()) ++ self.assertHeaders(resp.response, 'Account', 'POST') + + # Check if uploaded contents are completely deleted + self._check_contents_deleted(container_name) +diff --git a/tempest/api/object_storage/test_object_slo.py b/tempest/api/object_storage/test_object_slo.py +index c66776e4e..8bb2e6e4b 100644 +--- a/tempest/api/object_storage/test_object_slo.py ++++ b/tempest/api/object_storage/test_object_slo.py +@@ -17,7 +17,6 @@ import hashlib + from oslo_serialization import jsonutils as json + + from tempest.api.object_storage import base +-from tempest.common import custom_matchers + from tempest.common import utils + from tempest.lib.common.utils import data_utils + from tempest.lib.common.utils import test_utils +@@ -160,17 +159,7 @@ class ObjectSloTest(base.BaseObjectTest): + object_name, + params=params_del) + +- # When deleting SLO using multipart manifest, the response contains +- # not 'content-length' but 'transfer-encoding' header. This is the +- # special case, therefore the existence of response headers is checked +- # outside of custom matcher. +- self.assertIn('transfer-encoding', resp) +- self.assertIn('content-type', resp) +- self.assertIn('x-trans-id', resp) +- self.assertIn('date', resp) +- +- # Check only the format of common headers with custom matcher +- self.assertThat(resp, custom_matchers.AreAllWellFormatted()) ++ self.assertHeaders(resp, 'Object', 'DELETE') + + resp, body = self.container_client.list_container_objects( + self.container_name) +diff --git a/tempest/common/custom_matchers.py b/tempest/common/custom_matchers.py +index c702d8896..f1adeab64 100644 +--- a/tempest/common/custom_matchers.py ++++ b/tempest/common/custom_matchers.py +@@ -62,8 +62,9 @@ class ExistsAllResponseHeaders(object): + # [1] https://bugs.launchpad.net/swift/+bug/1537811 + # [2] http://tracker.ceph.com/issues/13582 + if ('content-length' not in actual and ++ 'transfer-encoding' not in actual and + self._content_length_required(actual)): +- return NonExistentHeader('content-length') ++ return NonExistentHeaders(['content-length', 'transfer-encoding']) + if 'content-type' not in actual: + return NonExistentHeader('content-type') + if 'x-trans-id' not in actual: +@@ -192,6 +193,19 @@ class NonExistentHeader(object): + return {} + + ++class NonExistentHeaders(object): ++ """Informs an error message in the case of missing certain headers""" ++ ++ def __init__(self, headers): ++ self.headers = headers ++ ++ def describe(self): ++ return "none of these headers exist: %s" % self.headers ++ ++ def get_details(self): ++ return {} ++ ++ + class InvalidHeaderValue(object): + """Informs an error message when a header contains a bad value""" + +-- +2.27.0 + |