Do not allow testcase skipping in refstack

Set the result of refstack_defcore as failed if there were skipped testcases
in a run.

Change-Id: Id782c2dee511cefbafd5b4440f386fd66f934a47
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>

3 files changed, 28 insertions, 0 deletions

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 672349cb5..ede9350ea 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -205,6 +205,7 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Defcore testcases.
diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 9749e361f..821d545bf 100644
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -94,6 +94,7 @@ class TempestCommon(singlevm.VmReady2):
                 'neutron_extensions']
         except Exception:  # pylint: disable=broad-except
             pass
+        self.deny_skipping = kwargs.get("deny_skipping", False)
 
     def check_services(self):
         """Check the mandatory services."""
@@ -663,3 +664,10 @@ class TempestCommon(singlevm.VmReady2):
         if self.flavor_alt:
             self.orig_cloud.delete_flavor(self.flavor_alt.id)
         super(TempestCommon, self).clean()
+
+    def is_successful(self):
+        """The overall result of the test."""
+        skips = self.details.get("skipped_number", 0)
+        if skips > 0 and self.deny_skipping:
+            return testcase.TestCase.EX_TESTCASE_FAILED
+        return super(TempestCommon, self).is_successful()
diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py
index de543b5d8..119959670 100644
--- a/functest/tests/unit/openstack/tempest/test_tempest.py
+++ b/functest/tests/unit/openstack/tempest/test_tempest.py
@@ -371,6 +371,25 @@ class OSTempestTesting(unittest.TestCase):
                 ['rally', 'verify', 'configure-verifier', '--reconfigure',
                  '--id', str(getattr(config.CONF, 'tempest_verifier_name'))])
 
+    def test_is_successful_false(self):
+        with mock.patch('six.moves.builtins.super') as mock_super:
+            self.tempestcommon.deny_skipping = True
+            self.tempestcommon.details = {"skipped_number": 2}
+            self.assertEqual(self.tempestcommon.is_successful(),
+                             testcase.TestCase.EX_TESTCASE_FAILED)
+            mock_super(tempest.TempestCommon,
+                       self).is_successful.assert_not_called()
+
+    def test_is_successful_true(self):
+        with mock.patch('six.moves.builtins.super') as mock_super:
+            self.tempestcommon.deny_skipping = False
+            self.tempestcommon.details = {"skipped_number": 2}
+            mock_super(tempest.TempestCommon,
+                       self).is_successful.return_value = 567
+            self.assertEqual(self.tempestcommon.is_successful(), 567)
+            mock_super(tempest.TempestCommon,
+                       self).is_successful.assert_called()
+
 
 if __name__ == "__main__":
     logging.disable(logging.CRITICAL)