diff options
| -rw-r--r-- | dovetail/compliance/debug.yml | 4 | ||||
| -rw-r--r-- | dovetail/conf/dovetail_config.yml | 20 | ||||
| -rw-r--r-- | dovetail/conf/functest_config.yml | 9 | ||||
| -rw-r--r-- | dovetail/report.py | 64 | ||||
| -rw-r--r-- | dovetail/testcase/example.tc001.yml | 2 | ||||
| -rw-r--r-- | dovetail/testcase/ipv6.tc001.yml | 2 | ||||
| -rw-r--r-- | dovetail/testcase/nfvi.tc101.yml | 8 | ||||
| -rw-r--r-- | dovetail/testcase/nfvi.tc102.yml | 8 |
8 files changed, 91 insertions, 26 deletions
diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml index 6fc4f0f1..bbcccd75 100644 --- a/dovetail/compliance/debug.yml +++ b/dovetail/compliance/debug.yml @@ -6,6 +6,8 @@ debug: name: debug testcases_list: - dovetail.example.tc002 - - dovetail.ipv6.tc001 + - dovetail.ipv6.tc008 + - dovetail.ipv6.tc009 - dovetail.nfvi.tc001 - dovetail.nfvi.tc002 + - dovetail.nfvi.tc101 diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index b6f7b016..f8f18e46 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -25,6 +25,20 @@ testarea_supported: - ipv6 - example +functest_testsuite: + - tempest_smoke_serial + - tempest_full_parallel + - rally_sanity + - promise + +functest_testcase: + - healthcheck + - vping_ssh + - vping_userdata + - doctor + - copper + - cloudify_ims + # used for testcase cmd template in jinja2 format # we have two variables available now # parameter path, use this path to walk through python object and get value @@ -53,3 +67,9 @@ validate_input: valid_docker_tag: - 'stable' - 'latest' + - 'colorado.1.0' + - 'colorado.2.0' + - 'colorado.3.0' + - 'danube.1.0' + - 'danube.2.0' + - 'danube.3.0' diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml index f20d1b7e..2c702cdb 100644 --- a/dovetail/conf/functest_config.yml +++ b/dovetail/conf/functest_config.yml @@ -4,18 +4,21 @@ functest: docker_tag: latest envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2 -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn - -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal' + -e BUILD_TAG=dovetail -e CI_DEBUG=true + -e DEPLOY_TYPE=baremetal + -e RESULTS_STORE=file:///home/opnfv/functest/results/functest_result.json' opts: '-id --privileged=true' pre_condition: - 'echo test for precondition in functest' cmds: - 'functest env prepare' - - 'functest testcase run {{validate_testcase}}' + - 'functest testcase run {{validate_testcase}} -r' post_condition: - 'echo test for postcondition in functest' result: dir: '/home/opnfv/functest/results' store_type: 'file' - file_path: 'tempest/tempest.log' + file_path: 'functest_result.json' + tp_path: 'tempest/tempest.log' db_url: 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1' creds: '/home/opnfv/functest/conf/openstack.creds' diff --git a/dovetail/report.py b/dovetail/report.py index 91a4ebd4..286c3893 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -221,7 +221,37 @@ class FunctestCrawler: self.logger.info('result file not found: %s', file_path) return None - try: + criteria = 'FAIL' + timestart = 0 + testcase_duration = 0 + testcase_name = testcase.validate_testcase() + json_results = {} + if testcase_name in dt_cfg.dovetail_config['functest_testcase']: + file_path = \ + os.path.join(dovetail_config['result_dir'], + dovetail_config[self.type]['result']['file_path']) + if not os.path.exists(file_path): + self.logger.info('result file not found: %s', file_path) + return None + with open(file_path, 'r') as f: + for jsonfile in f: + data = json.loads(jsonfile) + if testcase_name == data['case_name']: + criteria = data['details']['status'] + timestart = data['details']['timestart'] + testcase_duration = data['details']['duration'] + + json_results = {'criteria': criteria, + 'details': {"timestart": timestart, + "duration": testcase_duration, + "tests": '', "failures": ''}} + elif 'tempest' in testcase_name: + file_path = \ + os.path.join(dovetail_config['result_dir'], + dovetail_config[self.type]['result']['tp_path']) + if not os.path.exists(file_path): + self.logger.info('result file not found: %s', file_path) + return None with open(file_path, 'r') as myfile: output = myfile.read() error_logs = "" @@ -229,8 +259,7 @@ class FunctestCrawler: for match in re.findall('(.*?)[. ]*FAILED', output): error_logs += match - criteria = 'PASS' - failed_num = int(re.findall(' - Failed: (\d*)', output)[0]) + failed_num = int(re.findall(' - Failures: (\d*)', output)[0]) if failed_num != 0: criteria = 'FAIL' @@ -240,12 +269,9 @@ class FunctestCrawler: "duration": int(dur_sec_int), "tests": int(num_tests), "failures": failed_num, "errors": error_logs}} - self.logger.debug('Results: %s', str(json_results)) - return json_results - except Exception as e: - self.logger.error('Cannot read content from the file: %s, ' - 'exception: %s', file_path, e) - return None + + self.logger.debug('Results: %s', str(json_results)) + return json_results def crawl_from_url(self, testcase=None): url = \ @@ -289,17 +315,15 @@ class YardstickCrawler: if not os.path.exists(file_path): self.logger.info('result file not found: %s', file_path) return None - try: - with open(file_path, 'r') as myfile: - myfile.read() - criteria = 'PASS' - json_results = {'criteria': criteria} - self.logger.debug('Results: %s', str(json_results)) - return json_results - except Exception as e: - self.logger.error('Cannot read content from the file: %s, ' - 'exception: %s', file_path, e) - return None + criteria = 'FAIL' + with open(file_path, 'r') as f: + for jsonfile in f: + data = json.loads(jsonfile) + if 1 == data['status']: + criteria = 'PASS' + json_results = {'criteria': criteria} + self.logger.debug('Results: %s', str(json_results)) + return json_results def crawl_from_url(self, testcase=None): return None diff --git a/dovetail/testcase/example.tc001.yml b/dovetail/testcase/example.tc001.yml index e389a00f..0ba297a8 100644 --- a/dovetail/testcase/example.tc001.yml +++ b/dovetail/testcase/example.tc001.yml @@ -9,7 +9,7 @@ dovetail.example.tc001: - 'echo test for precondition' cmds: - 'functest env prepare' - - 'functest testcase run {{validate_testcase}}' + - 'functest testcase run {{validate_testcase}} -r' post_condition: - 'echo test for precondition' report: diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml index f9edf069..598e1cad 100644 --- a/dovetail/testcase/ipv6.tc001.yml +++ b/dovetail/testcase/ipv6.tc001.yml @@ -9,7 +9,7 @@ dovetail.ipv6.tc001: - 'echo test for precondition in testcase' cmds: - 'functest env prepare' - - 'functest testcase run {{validate_testcase}}' + - 'functest testcase run {{validate_testcase}} -r' post_condition: - 'echo test for precondition in testcase' report: diff --git a/dovetail/testcase/nfvi.tc101.yml b/dovetail/testcase/nfvi.tc101.yml new file mode 100644 index 00000000..7c8fb3ec --- /dev/null +++ b/dovetail/testcase/nfvi.tc101.yml @@ -0,0 +1,8 @@ +dovetail.nfvi.tc101: + name: dovetail.nfvi.tc101 + objective: measure number of cores and threads, available memory size and cache size + validate: + type: yardstick + testcase: opnfv_yardstick_tc001 + report: + sub_testcase_list: diff --git a/dovetail/testcase/nfvi.tc102.yml b/dovetail/testcase/nfvi.tc102.yml new file mode 100644 index 00000000..7ce0435e --- /dev/null +++ b/dovetail/testcase/nfvi.tc102.yml @@ -0,0 +1,8 @@ +dovetail.nfvi.tc102: + name: dovetail.nfvi.tc102 + objective: measure number of cores and threads, available memory size and cache size + validate: + type: yardstick + testcase: opnfv_yardstick_tc002 + report: + sub_testcase_list: |