aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJuha Kosonen <juha.kosonen@nokia.com>2019-03-18 13:40:05 +0200
committerCédric Ollivier <cedric.ollivier@orange.com>2019-03-19 14:07:11 +0100
commit376b8234ee7dc638b0025ee497fe5d0bda7e832a (patch)
tree1483dca90c75a5a5f796b0df9123fbd076e0a0c3
parent9035b678336ce221e9b28b53882ff5bbba5c28eb (diff)
Do not allow testcase skipping in refstack
Set the result of refstack_defcore as failed if there were skipped testcases in a run. Change-Id: Id782c2dee511cefbafd5b4440f386fd66f934a47 Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com> (cherry picked from commit cfd79ec4c39e215a25fc3d99fc5368e7d874f7bf)
-rw-r--r--docker/smoke/testcases.yaml1
-rw-r--r--functest/ci/testcases.yaml1
-rw-r--r--functest/opnfv_tests/openstack/tempest/tempest.py8
-rw-r--r--functest/tests/unit/openstack/tempest/test_tempest.py19
4 files changed, 29 insertions, 0 deletions
diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 67e157349..49ad8a8bd 100644
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -60,6 +60,7 @@ tiers:
project_name: functest
criteria: 100
blocking: false
+ deny_skipping: true
description: >-
This test case runs a group of Rally jobs used in
OpenStack gating
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 912f35b59..36a19c0f5 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -229,6 +229,7 @@ tiers:
project_name: functest
criteria: 100
blocking: false
+ deny_skipping: true
description: >-
This test case runs a group of Rally jobs used in
OpenStack gating
diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 597c711fa..57fedb79f 100644
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -84,6 +84,7 @@ class TempestCommon(singlevm.VmReady2):
'neutron_extensions']
except Exception: # pylint: disable=broad-except
pass
+ self.deny_skipping = kwargs.get("deny_skipping", False)
def check_services(self):
"""Check the mandatory services."""
@@ -488,3 +489,10 @@ class TempestCommon(singlevm.VmReady2):
if self.flavor_alt:
self.orig_cloud.delete_flavor(self.flavor_alt.id)
super(TempestCommon, self).clean()
+
+ def is_successful(self):
+ """The overall result of the test."""
+ skips = self.details.get("skipped_number", 0)
+ if skips > 0 and self.deny_skipping:
+ return testcase.TestCase.EX_TESTCASE_FAILED
+ return super(TempestCommon, self).is_successful()
diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py
index 646f88290..f5a2f410d 100644
--- a/functest/tests/unit/openstack/tempest/test_tempest.py
+++ b/functest/tests/unit/openstack/tempest/test_tempest.py
@@ -250,6 +250,25 @@ class OSTempestTesting(unittest.TestCase):
self._test_run(testcase.TestCase.EX_OK)
args[0].assert_called_once_with()
+ def test_is_successful_false(self):
+ with mock.patch('six.moves.builtins.super') as mock_super:
+ self.tempestcommon.deny_skipping = True
+ self.tempestcommon.details = {"skipped_number": 2}
+ self.assertEqual(self.tempestcommon.is_successful(),
+ testcase.TestCase.EX_TESTCASE_FAILED)
+ mock_super(tempest.TempestCommon,
+ self).is_successful.assert_not_called()
+
+ def test_is_successful_true(self):
+ with mock.patch('six.moves.builtins.super') as mock_super:
+ self.tempestcommon.deny_skipping = False
+ self.tempestcommon.details = {"skipped_number": 2}
+ mock_super(tempest.TempestCommon,
+ self).is_successful.return_value = 567
+ self.assertEqual(self.tempestcommon.is_successful(), 567)
+ mock_super(tempest.TempestCommon,
+ self).is_successful.assert_called()
+
if __name__ == "__main__":
logging.disable(logging.CRITICAL)