aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinda Wang <wangwulin@huawei.com>2017-03-15 09:53:37 +0000
committerLinda Wang <wangwulin@huawei.com>2017-03-15 10:44:37 +0000
commit2223051412c8855afebb7a9d657c90e9f235f78a (patch)
tree03b6260155dbb93973e4ec1fe7c2c6c204fe736a
parentc88fd9ac48589b4cb28a5ba5486adde1a21ae74c (diff)
Parse refstack_defcore results
JIRA: DOVETAIL-366 JIRA: FUNCTEST-758 Parse refstack_defcore results, including: 1. show number of tests passed, skipped and failed 2. record the detailed info about testcases passed, skipped and failed. Change-Id: I355a907c5c9ea2739f845447144c62613ec3bb6a Signed-off-by: Linda Wang <wangwulin@huawei.com>
-rwxr-xr-xfunctest/ci/testcases.yaml2
-rwxr-xr-xfunctest/opnfv_tests/openstack/refstack_client/refstack_client.py86
-rw-r--r--functest/opnfv_tests/openstack/tempest/conf_utils.py2
-rw-r--r--functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py11
4 files changed, 88 insertions, 13 deletions
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index bfbc3fdb0..bc7ac84ff 100755
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -132,7 +132,7 @@ tiers:
-
name: refstack_defcore
- criteria: 'success_rate == 100%'
+ criteria: 'success_rate == 80%'
blocking: false
clean_flag: false
description: >-
diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py
index d388dcd7d..c9f0f2752 100755
--- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py
+++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py
@@ -7,7 +7,10 @@
# http://www.apache.org/licenses/LICENSE-2.0
import argparse
import os
+import re
import sys
+import subprocess
+import time
from functest.core import testcase_base
from functest.opnfv_tests.openstack.tempest import conf_utils
@@ -24,6 +27,7 @@ class RefstackClient(testcase_base.TestcaseBase):
def __init__(self):
super(RefstackClient, self).__init__()
+ self.case_name = "refstack_defcore"
self.FUNCTEST_TEST = CONST.dir_functest_test
self.CONF_PATH = CONST.refstack_tempest_conf_path
self.DEFCORE_LIST = CONST.refstack_defcore_list
@@ -63,7 +67,80 @@ class RefstackClient(testcase_base.TestcaseBase):
"cd -;".format(CONST.dir_refstack_client,
self.confpath,
self.defcorelist))
- ft_utils.execute_command(cmd)
+ logger.info("Starting Refstack_defcore test case: '%s'." % cmd)
+
+ header = ("Tempest environment:\n"
+ " Installer: %s\n Scenario: %s\n Node: %s\n Date: %s\n" %
+ (CONST.INSTALLER_TYPE,
+ CONST.DEPLOY_SCENARIO,
+ CONST.NODE_NAME,
+ time.strftime("%a %b %d %H:%M:%S %Z %Y")))
+
+ f_stdout = open(
+ os.path.join(conf_utils.REFSTACK_RESULTS_DIR,
+ "refstack.log"), 'w+')
+ f_stderr = open(
+ os.path.join(conf_utils.REFSTACK_RESULTS_DIR,
+ "refstack-error.log"), 'w+')
+ f_env = open(os.path.join(conf_utils.REFSTACK_RESULTS_DIR,
+ "environment.log"), 'w+')
+ f_env.write(header)
+
+ p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+ stderr=f_stderr, bufsize=1)
+
+ with p.stdout:
+ for line in iter(p.stdout.readline, b''):
+ if 'Tests' in line:
+ break
+ logger.info(line.replace('\n', ''))
+ f_stdout.write(line)
+ p.wait()
+
+ f_stdout.close()
+ f_stderr.close()
+ f_env.close()
+
+ def parse_refstack_result(self):
+ try:
+ with open(os.path.join(conf_utils.REFSTACK_RESULTS_DIR,
+ "refstack.log"), 'r') as logfile:
+ output = logfile.read()
+
+ for match in re.findall("Ran: (\d+) tests in (\d+\.\d{4}) sec.",
+ output):
+ num_tests = match[0]
+ for match in re.findall("- Passed: (\d+)", output):
+ num_success = match
+ for match in re.findall("- Skipped: (\d+)", output):
+ num_skipped = match
+ for match in re.findall("- Failed: (\d+)", output):
+ num_failures = match
+ success_testcases = ""
+ for match in re.findall(r"\{0\}(.*?)[. ]*ok", output):
+ success_testcases += match + ", "
+ failed_testcases = ""
+ for match in re.findall(r"\{0\}(.*?)[. ]*FAILED", output):
+ failed_testcases += match + ", "
+ skipped_testcases = ""
+ for match in re.findall(r"\{0\}(.*?)[. ]*SKIPPED:", output):
+ skipped_testcases += match + ", "
+
+ num_executed = int(num_tests) - int(num_skipped)
+ success_rate = 100 * int(num_success) / int(num_executed)
+
+ self.details = {"num_tests": int(num_tests),
+ "num_failures": int(num_failures),
+ "success": success_testcases,
+ "failed": failed_testcases,
+ "skipped": skipped_testcases}
+ except Exception:
+ success_rate = 0
+
+ self.criteria = ft_utils.check_success_rate(
+ self.case_name, success_rate)
+ logger.info("Testcase %s success_rate is %s%%, is marked as %s"
+ % (self.case_name, success_rate, self.criteria))
def defcore_env_prepare(self):
try:
@@ -80,14 +157,21 @@ class RefstackClient(testcase_base.TestcaseBase):
return res
def run(self):
+ self.start_time = time.time()
+
+ if not os.path.exists(conf_utils.REFSTACK_RESULTS_DIR):
+ os.makedirs(conf_utils.REFSTACK_RESULTS_DIR)
+
try:
self.defcore_env_prepare()
self.run_defcore_default()
+ self.parse_refstack_result()
res = testcase_base.TestcaseBase.EX_OK
except Exception as e:
logger.error('Error with run: %s', e)
res = testcase_base.TestcaseBase.EX_RUN_ERROR
+ self.stop_time = time.time()
return res
def main(self, **kwargs):
diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py
index 185499749..a21322d86 100644
--- a/functest/opnfv_tests/openstack/tempest/conf_utils.py
+++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py
@@ -35,6 +35,8 @@ TEMPEST_DEFCORE = os.path.join(REPO_PATH, TEMPEST_TEST_LIST_DIR,
'defcore_req.txt')
TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
+REFSTACK_RESULTS_DIR = os.path.join(CONST.dir_results,
+ 'refstack')
CI_INSTALLER_TYPE = CONST.INSTALLER_TYPE
CI_INSTALLER_IP = CONST.INSTALLER_IP
diff --git a/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py b/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py
index 4eb5a2501..4e83f6bf3 100644
--- a/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py
+++ b/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py
@@ -38,17 +38,6 @@ class OSRefstackClientTesting(unittest.TestCase):
self.refstackclient.source_venv()
m.assert_any_call(cmd)
- def test_run_defcore_default(self):
- with mock.patch('functest.opnfv_tests.openstack.refstack_client.'
- 'refstack_client.ft_utils.execute_command') as m:
- cmd = ("cd {0};"
- "./refstack-client test -c {1} -v --test-list {2};"
- "cd -;".format(CONST.dir_refstack_client,
- self._config,
- self._testlist))
- self.refstackclient.run_defcore_default()
- m.assert_any_call(cmd)
-
def test_run_defcore(self):
config = 'tempest.conf'
testlist = 'testlist'