summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhongbo tian <hongbo.tianhongbo@huawei.com>2017-04-13 06:32:29 +0000
committerGerrit Code Review <gerrit@opnfv.org>2017-04-13 06:32:29 +0000
commit920281280fd055e796d8eec8b47416ec9a2d457f (patch)
treebc6e2d8e2402ac737a2e0c6147496e9d73cd12e6
parent52503b8bca1025a3412385b45591999530cedc40 (diff)
parent4d98856b87edbf784b12b375b5d30ab9f401d525 (diff)
Merge "dovetail tool: bugfix: test cases are erroneously judged"
-rw-r--r--dovetail/conf/functest_config.yml1
-rw-r--r--dovetail/conf/yardstick_config.yml1
-rw-r--r--dovetail/report.py120
-rwxr-xr-xdovetail/run.py27
-rw-r--r--dovetail/testcase.py9
-rw-r--r--dovetail/utils/dovetail_utils.py14
6 files changed, 97 insertions, 75 deletions
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index 5b215550..d1f1d6a0 100644
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -14,6 +14,5 @@ functest:
- 'echo test for postcondition in functest'
result:
dir: '/home/opnfv/functest/results'
- store_type: 'file'
file_path: 'dump.txt'
openrc: '/home/opnfv/functest/conf/openstack.creds'
diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml
index cb4b1c7b..e9a674a9 100644
--- a/dovetail/conf/yardstick_config.yml
+++ b/dovetail/conf/yardstick_config.yml
@@ -18,6 +18,5 @@ yardstick:
result:
dir: '/home/opnfv/yardstick/results'
log: '/tmp/yardstick'
- store_type: 'file'
file_path: 'yardstick.log'
openrc: '/etc/yardstick/openstack.creds'
diff --git a/dovetail/report.py b/dovetail/report.py
index 6c45e00c..dc53d307 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -19,6 +19,7 @@ from pbr import version
import utils.dovetail_logger as dt_logger
from utils.dovetail_config import DovetailConfig as dt_cfg
+import utils.dovetail_utils as dt_utils
from testcase import Testcase
@@ -208,76 +209,62 @@ class FunctestCrawler(object):
dt_logger.Logger(__name__ + '.FunctestCrawler').getLogger()
def crawl(self, testcase=None):
- store_type = \
- dt_cfg.dovetail_config[self.type]['result']['store_type']
- if store_type == 'file':
+ report_dest = dt_cfg.dovetail_config['report_dest']
+ if report_dest.lower() == 'file':
return self.crawl_from_file(testcase)
- if store_type == 'url':
+ if report_dest.lower().startswith('http'):
return self.crawl_from_url(testcase)
def crawl_from_file(self, testcase=None):
dovetail_config = dt_cfg.dovetail_config
criteria = 'FAIL'
+ details = {}
timestart = 0
- testcase_duration = 0
+ timestop = 0
+ duration = 0
testcase_name = testcase.validate_testcase()
- json_results = {}
+ file_path = \
+ os.path.join(dovetail_config['result_dir'],
+ dovetail_config[self.type]['result']['file_path'])
+ if not os.path.exists(file_path):
+ self.logger.info('result file not found: %s', file_path)
+ return None
if testcase_name in dt_cfg.dovetail_config['functest_testcase']:
- file_path = \
- os.path.join(dovetail_config['result_dir'],
- dovetail_config[self.type]['result']['file_path'])
- if not os.path.exists(file_path):
- self.logger.info('result file not found: %s', file_path)
- return None
- with open(file_path, 'r') as f:
- for jsonfile in f:
- try:
- data = json.loads(jsonfile)
- if testcase_name == data['case_name']:
- criteria = data['details']['status']
- timestart = data['details']['timestart']
- testcase_duration = data['details']['duration']
- except Exception:
- continue
-
- json_results = {'criteria': criteria,
- 'details': {"timestart": timestart,
- "duration": testcase_duration,
- "tests": '', "failures": ''}}
+ complex_testcase = False
elif testcase_name in dt_cfg.dovetail_config['functest_testsuite']:
- file_path = \
- os.path.join(dovetail_config['result_dir'],
- dovetail_config[self.type]['result']['file_path'])
- if not os.path.exists(file_path):
- self.logger.info('result file not found: %s', file_path)
- return None
- tests = 0
- failed_num = 0
- error_case = ''
- skipped_case = ''
- with open(file_path, 'r') as f:
- for jsonfile in f:
- try:
- data = json.loads(jsonfile)
- if testcase_name == data['case_name']:
- criteria = data['details']['status']
- timestart = data['details']['timestart']
- testcase_duration = data['details']['duration']
+ complex_testcase = True
+ else:
+ self.logger.error("Wrong Functest test case %s.", testcase_name)
+ return None
+ with open(file_path, 'r') as f:
+ for jsonfile in f:
+ try:
+ data = json.loads(jsonfile)
+ if testcase_name == data['case_name']:
+ criteria = data['criteria']
+ timestart = data['start_date']
+ timestop = data['stop_date']
+ duration = dt_utils.get_duration(timestart, timestop,
+ self.logger)
+ if complex_testcase:
tests = data['details']['tests']
failed_num = data['details']['failures']
error_case = data['details']['errors']
skipped_case = data['details']['skipped']
- except Exception:
- continue
-
- json_results = {'criteria': criteria,
- 'details': {"timestart": timestart,
- "duration": testcase_duration,
- "tests": tests,
- "failures": failed_num,
- "errors": error_case,
- "skipped": skipped_case}}
+ details = {"tests": tests,
+ "failures": failed_num,
+ "errors": error_case,
+ "skipped": skipped_case}
+ except KeyError as e:
+ self.logger.error("Key error, exception: %s", e)
+ return None
+ except ValueError:
+ continue
+
+ json_results = {'criteria': criteria, 'timestart': timestart,
+ 'timestop': timestop, 'duration': duration,
+ 'details': details}
self.logger.debug('Results: %s', str(json_results))
return json_results
@@ -310,12 +297,11 @@ class YardstickCrawler(object):
dt_logger.Logger(__name__ + '.YardstickCrawler').getLogger()
def crawl(self, testcase=None):
- store_type = \
- dt_cfg.dovetail_config[self.type]['result']['store_type']
- if store_type == 'file':
+ report_dest = dt_cfg.dovetail_config['report_dest']
+ if report_dest.lower() == 'file':
return self.crawl_from_file(testcase)
- if store_type == 'url':
+ if report_dest.lower().startswith('http'):
return self.crawl_from_url(testcase)
def crawl_from_file(self, testcase=None):
@@ -389,6 +375,15 @@ class FunctestChecker(object):
cls.logger = \
dt_logger.Logger(__name__ + '.FunctestChecker').getLogger()
+ @staticmethod
+ def get_sub_testcase(sub_testcase, result):
+ reg = sub_testcase + '[\s+\d+]'
+ find_reg = re.compile(reg)
+ match = find_reg.findall(result)
+ if match:
+ return True
+ return False
+
def check(self, testcase, db_result):
sub_testcase_list = testcase.sub_testcase()
@@ -406,10 +401,13 @@ class FunctestChecker(object):
testcase_passed = 'SKIP'
for sub_testcase in sub_testcase_list:
self.logger.debug('check sub_testcase:%s', sub_testcase)
- if sub_testcase in db_result['details']['errors']:
+ if self.get_sub_testcase(sub_testcase,
+ db_result['details']['errors']):
testcase.sub_testcase_passed(sub_testcase, 'FAIL')
testcase_passed = 'FAIL'
- elif sub_testcase in db_result['details']['skipped']:
+ continue
+ if self.get_sub_testcase(sub_testcase,
+ db_result['details']['skipped']):
testcase.sub_testcase_passed(sub_testcase, 'SKIP')
else:
testcase.sub_testcase_passed(sub_testcase, 'PASS')
diff --git a/dovetail/run.py b/dovetail/run.py
index 98a00085..5a902d7f 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -71,9 +71,14 @@ def run_test(testsuite, testarea, logger):
def check_tc_result(testcase, logger):
+ result_dir = dt_cfg.dovetail_config['result_dir']
+ validate_type = testcase.validate_type()
+ functest_result = dt_cfg.dovetail_config['functest']['result']['file_path']
if dt_cfg.dovetail_config['report_dest'].startswith("http"):
- if testcase.validate_type() == 'yardstick':
- logger.info("Results have been stored with files.")
+ if validate_type.lower() == 'yardstick':
+ logger.info("Results have been stored with file %s.",
+ os.path.join(result_dir,
+ testcase.validate_testcase() + '.out'))
else:
if dt_utils.check_db_results(dt_cfg.dovetail_config['report_dest'],
dt_cfg.dovetail_config['build_tag'],
@@ -83,9 +88,15 @@ def check_tc_result(testcase, logger):
else:
logger.error("Fail to push results to database.")
if dt_cfg.dovetail_config['report_dest'] == "file":
- logger.info("Results have been stored with files.")
- result = Report.get_result(testcase)
- Report.check_result(testcase, result)
+ if validate_type.lower() == 'yardstick':
+ logger.info("Results have been stored with file %s.",
+ os.path.join(result_dir,
+ testcase.validate_testcase() + '.out'))
+ if validate_type.lower() == 'functest':
+ logger.info("Results have been stored with file %s.",
+ os.path.join(result_dir, functest_result))
+ # result = Report.get_result(testcase)
+ # Report.check_result(testcase, result)
def validate_input(input_dict, check_dict, logger):
@@ -218,9 +229,9 @@ def main(*args, **kwargs):
if testsuite_validation and testarea_validation:
testsuite_yaml = load_testsuite(kwargs['testsuite'])
load_testcase()
- duration = run_test(testsuite_yaml, testarea, logger)
- if dt_cfg.dovetail_config['report_dest'] == "file":
- Report.generate(testsuite_yaml, testarea, duration)
+ run_test(testsuite_yaml, testarea, logger)
+ # if dt_cfg.dovetail_config['report_dest'] == "file":
+ # Report.generate(testsuite_yaml, testarea, duration)
else:
logger.error('invalid input commands, testsuite %s testarea %s',
kwargs['testsuite'], testarea)
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 21d655ef..13a31fde 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -150,13 +150,14 @@ class Testcase(object):
def mk_src_file(self):
testcase_src_file = self.testcase['validate']['pre_copy']['src_file']
try:
- with open(os.path.join(dt_cfg.dovetail_config['result_dir'],
- testcase_src_file), 'w+') as src_file:
+ file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
+ testcase_src_file)
+ with open(file_path, 'w+') as src_file:
if self.sub_testcase() is not None:
for sub_test in self.sub_testcase():
- self.logger.info('save testcases %s', sub_test)
+ self.logger.debug('save testcases %s', sub_test)
src_file.write(sub_test + '\n')
- self.logger.info('save testcases to %s', src_file)
+ self.logger.debug('save testcases to %s', file_path)
except Exception:
self.logger.error('Failed to save: %s', src_file)
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 766bb646..fa99e004 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -16,6 +16,7 @@ import subprocess
from collections import Mapping, Set, Sequence
import json
import urllib2
+from datetime import datetime
def exec_log(verbose, logger, msg, level, flush=False):
@@ -144,6 +145,19 @@ def check_db_results(db_url, build_tag, testcase, logger):
return False
+def get_duration(start_date, stop_date, logger):
+ fmt = '%Y-%m-%d %H:%M:%S'
+ try:
+ datetime_start = datetime.strptime(start_date, fmt)
+ datetime_stop = datetime.strptime(stop_date, fmt)
+ delta = (datetime_stop - datetime_start).seconds
+ res = "%sm%ss" % (delta / 60, delta % 60)
+ return res
+ except ValueError as e:
+ logger.error("ValueError: %s", e)
+ return None
+
+
def show_progress_bar(length):
max_len = 50
length %= max_len