summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMatthewLi <matthew.lijun@huawei.com>2017-01-19 02:07:16 -0500
committerMatthewLi <matthew.lijun@huawei.com>2017-01-25 20:38:24 -0500
commit3374309f5aae1dc2be3ac1eee545a05bca69b8e6 (patch)
tree93f118cf97b72faf1dd26561441d9e4dced98258
parent8475a7b2d96bc55da4414220337315c7096ad686 (diff)
dovetail tool: functest different testsuite support and yardstick testcase added
JIRA: DOVETAIL-186 Change-Id: Id0eec6652c9595c660436dec89a309a77b16bc03 Signed-off-by: MatthewLi <matthew.lijun@huawei.com>
-rw-r--r--dovetail/compliance/debug.yml4
-rw-r--r--dovetail/conf/dovetail_config.yml20
-rw-r--r--dovetail/conf/functest_config.yml9
-rw-r--r--dovetail/report.py64
-rw-r--r--dovetail/testcase/example.tc001.yml2
-rw-r--r--dovetail/testcase/ipv6.tc001.yml2
-rw-r--r--dovetail/testcase/nfvi.tc101.yml8
-rw-r--r--dovetail/testcase/nfvi.tc102.yml8
8 files changed, 91 insertions, 26 deletions
diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml
index 6fc4f0f1..bbcccd75 100644
--- a/dovetail/compliance/debug.yml
+++ b/dovetail/compliance/debug.yml
@@ -6,6 +6,8 @@ debug:
name: debug
testcases_list:
- dovetail.example.tc002
- - dovetail.ipv6.tc001
+ - dovetail.ipv6.tc008
+ - dovetail.ipv6.tc009
- dovetail.nfvi.tc001
- dovetail.nfvi.tc002
+ - dovetail.nfvi.tc101
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index b6f7b016..f8f18e46 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -25,6 +25,20 @@ testarea_supported:
- ipv6
- example
+functest_testsuite:
+ - tempest_smoke_serial
+ - tempest_full_parallel
+ - rally_sanity
+ - promise
+
+functest_testcase:
+ - healthcheck
+ - vping_ssh
+ - vping_userdata
+ - doctor
+ - copper
+ - cloudify_ims
+
# used for testcase cmd template in jinja2 format
# we have two variables available now
# parameter path, use this path to walk through python object and get value
@@ -53,3 +67,9 @@ validate_input:
valid_docker_tag:
- 'stable'
- 'latest'
+ - 'colorado.1.0'
+ - 'colorado.2.0'
+ - 'colorado.3.0'
+ - 'danube.1.0'
+ - 'danube.2.0'
+ - 'danube.3.0'
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index f20d1b7e..2c702cdb 100644
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -4,18 +4,21 @@ functest:
docker_tag: latest
envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2
-e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn
- -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal'
+ -e BUILD_TAG=dovetail -e CI_DEBUG=true
+ -e DEPLOY_TYPE=baremetal
+ -e RESULTS_STORE=file:///home/opnfv/functest/results/functest_result.json'
opts: '-id --privileged=true'
pre_condition:
- 'echo test for precondition in functest'
cmds:
- 'functest env prepare'
- - 'functest testcase run {{validate_testcase}}'
+ - 'functest testcase run {{validate_testcase}} -r'
post_condition:
- 'echo test for postcondition in functest'
result:
dir: '/home/opnfv/functest/results'
store_type: 'file'
- file_path: 'tempest/tempest.log'
+ file_path: 'functest_result.json'
+ tp_path: 'tempest/tempest.log'
db_url: 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1'
creds: '/home/opnfv/functest/conf/openstack.creds'
diff --git a/dovetail/report.py b/dovetail/report.py
index 91a4ebd4..286c3893 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -221,7 +221,37 @@ class FunctestCrawler:
self.logger.info('result file not found: %s', file_path)
return None
- try:
+ criteria = 'FAIL'
+ timestart = 0
+ testcase_duration = 0
+ testcase_name = testcase.validate_testcase()
+ json_results = {}
+ if testcase_name in dt_cfg.dovetail_config['functest_testcase']:
+ file_path = \
+ os.path.join(dovetail_config['result_dir'],
+ dovetail_config[self.type]['result']['file_path'])
+ if not os.path.exists(file_path):
+ self.logger.info('result file not found: %s', file_path)
+ return None
+ with open(file_path, 'r') as f:
+ for jsonfile in f:
+ data = json.loads(jsonfile)
+ if testcase_name == data['case_name']:
+ criteria = data['details']['status']
+ timestart = data['details']['timestart']
+ testcase_duration = data['details']['duration']
+
+ json_results = {'criteria': criteria,
+ 'details': {"timestart": timestart,
+ "duration": testcase_duration,
+ "tests": '', "failures": ''}}
+ elif 'tempest' in testcase_name:
+ file_path = \
+ os.path.join(dovetail_config['result_dir'],
+ dovetail_config[self.type]['result']['tp_path'])
+ if not os.path.exists(file_path):
+ self.logger.info('result file not found: %s', file_path)
+ return None
with open(file_path, 'r') as myfile:
output = myfile.read()
error_logs = ""
@@ -229,8 +259,7 @@ class FunctestCrawler:
for match in re.findall('(.*?)[. ]*FAILED', output):
error_logs += match
- criteria = 'PASS'
- failed_num = int(re.findall(' - Failed: (\d*)', output)[0])
+ failed_num = int(re.findall(' - Failures: (\d*)', output)[0])
if failed_num != 0:
criteria = 'FAIL'
@@ -240,12 +269,9 @@ class FunctestCrawler:
"duration": int(dur_sec_int),
"tests": int(num_tests), "failures": failed_num,
"errors": error_logs}}
- self.logger.debug('Results: %s', str(json_results))
- return json_results
- except Exception as e:
- self.logger.error('Cannot read content from the file: %s, '
- 'exception: %s', file_path, e)
- return None
+
+ self.logger.debug('Results: %s', str(json_results))
+ return json_results
def crawl_from_url(self, testcase=None):
url = \
@@ -289,17 +315,15 @@ class YardstickCrawler:
if not os.path.exists(file_path):
self.logger.info('result file not found: %s', file_path)
return None
- try:
- with open(file_path, 'r') as myfile:
- myfile.read()
- criteria = 'PASS'
- json_results = {'criteria': criteria}
- self.logger.debug('Results: %s', str(json_results))
- return json_results
- except Exception as e:
- self.logger.error('Cannot read content from the file: %s, '
- 'exception: %s', file_path, e)
- return None
+ criteria = 'FAIL'
+ with open(file_path, 'r') as f:
+ for jsonfile in f:
+ data = json.loads(jsonfile)
+ if 1 == data['status']:
+ criteria = 'PASS'
+ json_results = {'criteria': criteria}
+ self.logger.debug('Results: %s', str(json_results))
+ return json_results
def crawl_from_url(self, testcase=None):
return None
diff --git a/dovetail/testcase/example.tc001.yml b/dovetail/testcase/example.tc001.yml
index e389a00f..0ba297a8 100644
--- a/dovetail/testcase/example.tc001.yml
+++ b/dovetail/testcase/example.tc001.yml
@@ -9,7 +9,7 @@ dovetail.example.tc001:
- 'echo test for precondition'
cmds:
- 'functest env prepare'
- - 'functest testcase run {{validate_testcase}}'
+ - 'functest testcase run {{validate_testcase}} -r'
post_condition:
- 'echo test for precondition'
report:
diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml
index f9edf069..598e1cad 100644
--- a/dovetail/testcase/ipv6.tc001.yml
+++ b/dovetail/testcase/ipv6.tc001.yml
@@ -9,7 +9,7 @@ dovetail.ipv6.tc001:
- 'echo test for precondition in testcase'
cmds:
- 'functest env prepare'
- - 'functest testcase run {{validate_testcase}}'
+ - 'functest testcase run {{validate_testcase}} -r'
post_condition:
- 'echo test for precondition in testcase'
report:
diff --git a/dovetail/testcase/nfvi.tc101.yml b/dovetail/testcase/nfvi.tc101.yml
new file mode 100644
index 00000000..7c8fb3ec
--- /dev/null
+++ b/dovetail/testcase/nfvi.tc101.yml
@@ -0,0 +1,8 @@
+dovetail.nfvi.tc101:
+ name: dovetail.nfvi.tc101
+ objective: measure number of cores and threads, available memory size and cache size
+ validate:
+ type: yardstick
+ testcase: opnfv_yardstick_tc001
+ report:
+ sub_testcase_list:
diff --git a/dovetail/testcase/nfvi.tc102.yml b/dovetail/testcase/nfvi.tc102.yml
new file mode 100644
index 00000000..7ce0435e
--- /dev/null
+++ b/dovetail/testcase/nfvi.tc102.yml
@@ -0,0 +1,8 @@
+dovetail.nfvi.tc102:
+ name: dovetail.nfvi.tc102
+ objective: measure number of cores and threads, available memory size and cache size
+ validate:
+ type: yardstick
+ testcase: opnfv_yardstick_tc002
+ report:
+ sub_testcase_list: