From ab26ce1cc28c590115ac07ea4a51b9f75f902c9a Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Tue, 14 Apr 2020 11:34:01 +0200 Subject: Backport Fixes-race-condition-in-test_add_remove_fixed_ip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It has sometimes failed in gates and Orange reported the issue. Backporting is the best approach as for swift testing. Change-Id: I7c66faf77cdee3f52a2299d64dc4140de22f4cad Signed-off-by: Cédric Ollivier --- docker/core/Dockerfile | 5 +- ...ace-condition-in-test_add_remove_fixed_ip.patch | 165 +++++++++++++++++++++ 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 docker/core/Fixes-race-condition-in-test_add_remove_fixed_ip.patch (limited to 'docker/core') diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile index 9415c1270..6c8165a74 100644 --- a/docker/core/Dockerfile +++ b/docker/core/Dockerfile @@ -4,6 +4,7 @@ ARG BRANCH=stable/jerma ARG OPENSTACK_TAG=stable/train COPY Accept-custom-registered-endpoints.patch /tmp/Accept-custom-registered-endpoints.patch +COPY Fixes-race-condition-in-test_add_remove_fixed_ip.patch /tmp/Fixes-race-condition-in-test_add_remove_fixed_ip.patch RUN apk --no-cache add --update \ python3 libffi openssl libjpeg-turbo py3-pip bash \ grep sed wget ca-certificates git openssh-client qemu-img iputils coreutils mailcap && \ @@ -40,7 +41,9 @@ RUN apk --no-cache add --update \ git config --global user.email "opnfv-tech-discuss@lists.opnfv.org" && \ git config --global user.name "Functest" && \ patch -p1 < /tmp/Accept-custom-registered-endpoints.patch && \ - git commit -a -m "Apply Accept-custom-registered-endpoints" && \ + patch -p1 < /tmp/Fixes-race-condition-in-test_add_remove_fixed_ip.patch && \ + git commit -a -m "Backport critical bugfixes" && \ rm ~/.gitconfig) && \ rm /tmp/Accept-custom-registered-endpoints.patch && \ + rm /tmp/Fixes-race-condition-in-test_add_remove_fixed_ip.patch && \ apk del .build-deps diff --git a/docker/core/Fixes-race-condition-in-test_add_remove_fixed_ip.patch b/docker/core/Fixes-race-condition-in-test_add_remove_fixed_ip.patch new file mode 100644 index 000000000..a9f3c7e6d --- /dev/null +++ b/docker/core/Fixes-race-condition-in-test_add_remove_fixed_ip.patch @@ -0,0 +1,165 @@ +From 61a3c8efa4a5e41dc6b5fd2d7a28a25555ebb54b Mon Sep 17 00:00:00 2001 +From: David Sedlák +Date: Wed, 30 Oct 2019 15:38:21 +0100 +Subject: [PATCH] Fixes race condition in test_add_remove_fixed_ip + +Currently race condition can occure in +tempest.api.compute.servers.test_attach_interfaces. +AttachInterfacesUnderV243Test.test_add_remove_fixed_ip +when floating IP added during resource preparation doesn't appear in +the list of original IPs that is created at the beggining of the test, +which then confuses the test +and floating ip is later recognized as fixed IP added in the test. +more details including log: +https://bugzilla.redhat.com/show_bug.cgi?id=1752416 + +This change ensures possible floating IP added during server +creation is always present in the set of original IPs and also +during every comparasion of IPs. + +Closes-Bug: #1866179 + +Change-Id: Ic3a3e0708714b6d6c9c226e641e1c520e5ebde9d +Signed-off-by: David Sedlák +--- + +diff --git a/tempest/api/compute/servers/test_attach_interfaces.py b/tempest/api/compute/servers/test_attach_interfaces.py +index df8da07..c1af6c7 100644 +--- a/tempest/api/compute/servers/test_attach_interfaces.py ++++ b/tempest/api/compute/servers/test_attach_interfaces.py +@@ -86,12 +86,16 @@ + # apparently not enough? Add cleanup here. + self.addCleanup(self.delete_server, server['id']) + self._wait_for_validation(server, validation_resources) ++ try: ++ fip = set([validation_resources['floating_ip']['ip']]) ++ except KeyError: ++ fip = () + ifs = (self.interfaces_client.list_interfaces(server['id']) + ['interfaceAttachments']) + body = waiters.wait_for_interface_status( + self.interfaces_client, server['id'], ifs[0]['port_id'], 'ACTIVE') + ifs[0]['port_state'] = body['port_state'] +- return server, ifs ++ return server, ifs, fip + + + class AttachInterfacesTestJSON(AttachInterfacesTestBase): +@@ -226,7 +230,7 @@ + @decorators.idempotent_id('73fe8f02-590d-4bf1-b184-e9ca81065051') + @utils.services('network') + def test_create_list_show_delete_interfaces_by_network_port(self): +- server, ifs = self._create_server_get_interfaces() ++ server, ifs, _ = self._create_server_get_interfaces() + interface_count = len(ifs) + self.assertGreater(interface_count, 0) + +@@ -268,7 +272,7 @@ + raise self.skipException("Only owner network supports " + "creating interface by fixed ip.") + +- server, ifs = self._create_server_get_interfaces() ++ server, ifs, _ = self._create_server_get_interfaces() + interface_count = len(ifs) + self.assertGreater(interface_count, 0) + +@@ -354,9 +358,8 @@ + not CONF.network.shared_physical_network): + raise self.skipException("Only owner network supports " + "creating interface by fixed ip.") +- + # Add and Remove the fixed IP to server. +- server, ifs = self._create_server_get_interfaces() ++ server, ifs, fip = self._create_server_get_interfaces() + original_interface_count = len(ifs) # This is the number of ports. + self.assertGreater(original_interface_count, 0) + # Get the starting list of IPs on the server. +@@ -369,6 +372,9 @@ + self.assertEqual(1, len(addresses), addresses) # number of networks + # Keep track of the original addresses so we can know which IP is new. + original_ips = [addr['addr'] for addr in list(addresses.values())[0]] ++ # Make sure the floating IP possibly assigned during ++ # server creation is always present in the set of original ips. ++ original_ips = set(original_ips).union(fip) + original_ip_count = len(original_ips) + self.assertGreater(original_ip_count, 0, addresses) # at least 1 + network_id = ifs[0]['net_id'] +@@ -376,40 +382,22 @@ + # fixed IP on the same network (and same port since we only have one + # port). + self.servers_client.add_fixed_ip(server['id'], networkId=network_id) +- # Wait for the ips count to increase by one. + +- def _get_server_floating_ips(): +- _floating_ips = [] +- _server = self.os_primary.servers_client.show_server( +- server['id'])['server'] +- for _ip_set in _server['addresses']: +- for _ip in _server['addresses'][_ip_set]: +- if _ip['OS-EXT-IPS:type'] == 'floating': +- _floating_ips.append(_ip['addr']) +- return _floating_ips +- +- def _wait_for_ip_increase(): ++ def _wait_for_ip_change(expected_count): + _addresses = self.os_primary.servers_client.list_addresses( + server['id'])['addresses'] +- _ips = [addr['addr'] for addr in list(_addresses.values())[0]] +- LOG.debug("Wait for IP increase. All IPs still associated to " ++ _ips = set([addr['addr'] for addr in list(_addresses.values())[0]]) ++ # Make sure possible floating ip is always present in the set. ++ _ips = _ips.union(fip) ++ LOG.debug("Wait for change of IPs. All IPs still associated to " + "the server %(id)s: %(ips)s", + {'id': server['id'], 'ips': _ips}) +- if len(_ips) == original_ip_count + 1: +- return True +- elif len(_ips) == original_ip_count: +- return False +- # If not, lets remove any floating IP from the list and check again +- _fips = _get_server_floating_ips() +- _ips = [_ip for _ip in _ips if _ip not in _fips] +- LOG.debug("Wait for IP increase. Fixed IPs still associated to " +- "the server %(id)s: %(ips)s", +- {'id': server['id'], 'ips': _ips}) +- return len(_ips) == original_ip_count + 1 ++ return len(_ips) == expected_count + ++ # Wait for the ips count to increase by one. + if not test_utils.call_until_true( +- _wait_for_ip_increase, CONF.compute.build_timeout, +- CONF.compute.build_interval): ++ _wait_for_ip_change, CONF.compute.build_timeout, ++ CONF.compute.build_interval, original_ip_count + 1): + raise lib_exc.TimeoutException( + 'Timed out while waiting for IP count to increase.') + +@@ -428,26 +416,8 @@ + break + self.servers_client.remove_fixed_ip(server['id'], address=fixed_ip) + # Wait for the interface count to decrease by one. +- +- def _wait_for_ip_decrease(): +- _addresses = self.os_primary.servers_client.list_addresses( +- server['id'])['addresses'] +- _ips = [addr['addr'] for addr in list(_addresses.values())[0]] +- LOG.debug("Wait for IP decrease. All IPs still associated to " +- "the server %(id)s: %(ips)s", +- {'id': server['id'], 'ips': _ips}) +- if len(_ips) == original_ip_count: +- return True +- # If not, lets remove any floating IP from the list and check again +- _fips = _get_server_floating_ips() +- _ips = [_ip for _ip in _ips if _ip not in _fips] +- LOG.debug("Wait for IP decrease. Fixed IPs still associated to " +- "the server %(id)s: %(ips)s", +- {'id': server['id'], 'ips': _ips}) +- return len(_ips) == original_ip_count +- + if not test_utils.call_until_true( +- _wait_for_ip_decrease, CONF.compute.build_timeout, +- CONF.compute.build_interval): ++ _wait_for_ip_change, CONF.compute.build_timeout, ++ CONF.compute.build_interval, original_ip_count): + raise lib_exc.TimeoutException( + 'Timed out while waiting for IP count to decrease.') -- cgit 1.2.3-korg