diff options
| -rw-r--r-- | docker/Dockerfile | 6 | ||||
| -rw-r--r-- | dovetail/compliance/debug.yml | 9 | ||||
| -rw-r--r-- | dovetail/conf/dovetail_config.py | 9 | ||||
| -rw-r--r-- | dovetail/conf/dovetail_config.yml | 2 | ||||
| -rw-r--r-- | dovetail/report.py | 137 | ||||
| -rwxr-xr-x | dovetail/run.py | 35 | ||||
| -rw-r--r-- | dovetail/tests/unit/test_parser.py | 3 | ||||
| -rw-r--r-- | dovetail/utils/dovetail_logger.py | 10 | ||||
| -rw-r--r-- | dovetail/utils/dovetail_utils.py | 2 | ||||
| -rw-r--r-- | ez_setup.py | 5 |
10 files changed, 146 insertions, 72 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index cc21203d..19dc3619 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,8 @@ FROM ubuntu:14.04 MAINTAINER Leo Wang <grakiss.wanglei@huawei.com> LABEL version="0.1" description="OPNFV Dovetail Docker Container" +ARG BRANCH=master + RUN \ apt-get update \ && \ @@ -17,7 +19,7 @@ RUN \ && \ echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list \ && \ - apt-get update && apt-get install -y docker-engine=1.9.1-0~trusty \ + apt-get update && apt-get install -y docker-engine=1.12.3-0~trusty \ && \ pip install pyyaml \ click \ @@ -30,7 +32,7 @@ WORKDIR /home/opnfv RUN \ git config --global http.sslVerify false \ && \ - git clone https://gerrit.opnfv.org/gerrit/dovetail.git ${REPOS_DIR} \ + git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/dovetail.git ${REPOS_DIR} \ && \ mkdir -p ${REPOS_DIR}/results diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml new file mode 100644 index 00000000..a0a2d3ee --- /dev/null +++ b/dovetail/compliance/debug.yml @@ -0,0 +1,9 @@ +# only used for dovetail tool development debug +# nfvi.tc001 tc002 running time is shorter, about 3 minutes +# ipv6.tc001 about 20 minutes +debug: + name: debug + testcases_list: + - dovetail.ipv6.tc001 + - dovetail.nfvi.tc001 + - dovetail.nfvi.tc002 diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py index c37e8d07..14dc59a3 100644 --- a/dovetail/conf/dovetail_config.py +++ b/dovetail/conf/dovetail_config.py @@ -17,7 +17,7 @@ class DovetailConfig: COMPLIANCE_PATH = './compliance/' TESTCASE_PATH = './testcase/' # testsuite supported tuple, should adjust accordingly - testsuite_supported = ('compliance_set', 'proposed_tests') + testsuite_supported = ('compliance_set', 'proposed_tests', 'debug') # testarea supported tuple, should adjust accordingly testarea_supported = ('vimops', 'nfvi', 'ipv6') @@ -51,9 +51,10 @@ class DovetailConfig: @classmethod def update_envs(cls, options): for item in options: - if options[item] is not None: - key = cls.cmd_name_trans(item) - os.environ[key] = options[item] + key = cls.cmd_name_trans(item) + if not options[item] and key in os.environ: + options[item] = os.environ[key] + if options[item]: cls.update_config_envs('functest', key) cls.update_config_envs('yardstick', key) diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index 1f5de672..c4131a91 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -3,6 +3,8 @@ work_dir: /home/opnfv/dovetail result_dir: /home/opnfv/dovetail/results report_file: 'dovetail_report.txt' cli_file_name: 'cmd_config.yml' +# TO DO: once version scheme settled, adjust accordingly +repo: 'https://github.com/opnfv/dovetail/tree/master/' # used for testcase cmd template in jinja2 format # we have two variables available now diff --git a/dovetail/report.py b/dovetail/report.py index 52c8ebf9..7fd4076d 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -6,14 +6,18 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 # +from __future__ import division + import json import urllib2 import re import os +import datetime +import uuid import utils.dovetail_logger as dt_logger -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg from testcase import Testcase @@ -40,9 +44,18 @@ class Report: checker.check(testcase, db_result) @classmethod - def generate_json(cls, testsuite_yaml, testarea): + def generate_json(cls, testsuite_yaml, testarea, duration): report_obj = {} + # TO DO: once version scheme settled, adjust accordingly + report_obj['version'] = '1.0' report_obj['testsuite'] = testsuite_yaml['name'] + # TO DO: once dashboard url settled, adjust accordingly + report_obj['dashboard'] = None + report_obj['validation_ID'] = str(uuid.uuid4()) + report_obj['upload_date'] =\ + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + report_obj['duration'] = duration + report_obj['testcases_list'] = [] testarea_list = [] for value in testsuite_yaml['testcases_list']: @@ -50,61 +63,97 @@ class Report: testarea_list.append(value) for testcase_name in testarea_list: testcase = Testcase.get(testcase_name) - testcase_in_rpt = {} - testcase_in_rpt['name'] = testcase_name + testcase_inreport = {} + testcase_inreport['name'] = testcase_name if testcase is None: - testcase_in_rpt['result'] = 'Undefined' - testcase_in_rpt['objective'] = '' - testcase_in_rpt['sub_testcase'] = [] - report_obj['testcases_list'].append(testcase_in_rpt) + testcase_inreport['result'] = 'Undefined' + testcase_inreport['objective'] = '' + testcase_inreport['sub_testcase'] = [] + report_obj['testcases_list'].append(testcase_inreport) continue - testcase_in_rpt['result'] = get_pass_str(testcase.passed()) - testcase_in_rpt['objective'] = testcase.objective() - testcase_in_rpt['sub_testcase'] = [] + testcase_inreport['result'] = get_pass_str(testcase.passed()) + testcase_inreport['objective'] = testcase.objective() + testcase_inreport['sub_testcase'] = [] if testcase.sub_testcase() is not None: for sub_test in testcase.sub_testcase(): - testcase_in_rpt['sub_testcase'].append({ + testcase_inreport['sub_testcase'].append({ 'name': sub_test, 'result': get_pass_str( testcase.sub_testcase_passed(sub_test)) }) - report_obj['testcases_list'].append(testcase_in_rpt) + report_obj['testcases_list'].append(testcase_inreport) cls.logger.info(json.dumps(report_obj)) return report_obj @classmethod - def generate(cls, testsuite_yaml, testarea): - rpt_data = cls.generate_json(testsuite_yaml, testarea) - rpt_text = '' - split_line = '+-----------------------------------------------------' - split_line += '---------------------+\n' - - rpt_text += '\n\ -+==========================================================================+\n\ -| report |\n' - rpt_text += split_line - rpt_text += '|testsuite: %s\n' % rpt_data['testsuite'] - for testcase in rpt_data['testcases_list']: - rpt_text += '| [testcase]: %s\t\t\t\t[%s]\n' % \ - (testcase['name'], testcase['result']) - rpt_text += '| |-objective: %s\n' % testcase['objective'] - if 'sub_testcase' in testcase: - for sub_test in testcase['sub_testcase']: - rpt_text += '| |-%s \t\t [%s]\n' % \ - (sub_test['name'], sub_test['result']) - rpt_text += split_line - - cls.logger.info(rpt_text) - cls.save(rpt_text) - return rpt_text + def generate(cls, testsuite_yaml, testarea, duration): + report_data = cls.generate_json(testsuite_yaml, testarea, duration) + report_txt = '' + report_txt += '\n\nDovetail Report\n' + report_txt += 'Version: %s\n' % report_data['version'] + report_txt += 'TestSuite: %s\n' % report_data['testsuite'] + report_txt += 'Result Dashboard: %s\n' % report_data['dashboard'] + report_txt += 'Validation ID: %s\n' % report_data['validation_ID'] + report_txt += 'Upload Date: %s\n' % report_data['upload_date'] + if report_data['duration'] == 0: + report_txt += 'Duration: %s\n\n' % 'NA' + else: + report_txt += 'Duration: %.2f s\n\n' % report_data['duration'] + + total_num = 0 + pass_num = 0 + sub_report = {} + testcase_num = {} + testcase_passnum = {} + for area in dt_cfg.testarea_supported: + sub_report[area] = '' + testcase_num[area] = 0 + testcase_passnum[area] = 0 + + # TO DO: once version scheme settled, adjust accordingly + spec_link = dt_cfg.dovetail_config['repo'] + 'dovetail/testcase' + for testcase in report_data['testcases_list']: + pattern = re.compile('|'.join(dt_cfg.testarea_supported)) + area = pattern.findall(testcase['name'])[0] + result_dir = dt_cfg.dovetail_config['result_dir'] + sub_report[area] += '- <%s> %s result: <%s>\n' %\ + (spec_link, testcase['name'], result_dir) + testcase_num[area] += 1 + total_num += 1 + if testcase['result'] == 'PASS': + testcase_passnum[area] += 1 + pass_num += 1 + + if total_num != 0: + pass_rate = pass_num/total_num + report_txt += 'Pass Rate: %.2f%% (%s/%s)\n' %\ + (pass_rate*100, pass_num, total_num) + report_txt += 'Assessed test areas:\n' + for key in sub_report: + if testcase_num[key] != 0: + pass_rate = testcase_passnum[key]/testcase_num[key] + # TO DO: once version scheme settled, adjust accordingly + doc_link = dt_cfg.dovetail_config['repo'] +\ + ('docs/testsuites/%s' % key) + report_txt += '- %s results: <%s> pass %.2f%%\n' %\ + (key, doc_link, pass_rate*100) + for key in sub_report: + if testcase_num[key] != 0: + pass_rate = testcase_passnum[key]/testcase_num[key] + report_txt += '%s: pass rate %.2f%%\n' % (key, pass_rate*100) + report_txt += sub_report[key] + + cls.logger.info(report_txt) + cls.save(report_txt) + return report_txt # save to disk as default @classmethod def save(cls, report): - report_file_name = dt_config.dovetail_config['report_file'] + report_file_name = dt_cfg.dovetail_config['report_file'] try: - with open(os.path.join(dt_config.dovetail_config['result_dir'], + with open(os.path.join(dt_cfg.dovetail_config['result_dir'], report_file_name), 'w') as report_file: report_file.write(report) cls.logger.info('save report to %s' % report_file_name) @@ -160,7 +209,7 @@ class FunctestCrawler: def crawl(self, testcase=None): store_type = \ - dt_config.dovetail_config[self.type]['result']['store_type'] + dt_cfg.dovetail_config[self.type]['result']['store_type'] if store_type == 'file': return self.crawl_from_file(testcase) @@ -168,7 +217,7 @@ class FunctestCrawler: return self.crawl_from_url(testcase) def crawl_from_file(self, testcase=None): - dovetail_config = dt_config.dovetail_config + dovetail_config = dt_cfg.dovetail_config file_path = \ os.path.join(dovetail_config['result_dir'], dovetail_config[self.type]['result']['file_path']) @@ -204,7 +253,7 @@ class FunctestCrawler: def crawl_from_url(self, testcase=None): url = \ - dt_config.dovetail_config[self.type]['result']['db_url'] % testcase + dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase self.logger.debug("Query to rest api: %s" % url) try: data = json.load(urllib2.urlopen(url)) @@ -228,7 +277,7 @@ class YardstickCrawler: def crawl(self, testcase=None): store_type = \ - dt_config.dovetail_config[self.type]['result']['store_type'] + dt_cfg.dovetail_config[self.type]['result']['store_type'] if store_type == 'file': return self.crawl_from_file(testcase) @@ -236,7 +285,7 @@ class YardstickCrawler: return self.crawl_from_url(testcase) def crawl_from_file(self, testcase=None): - file_path = os.path.join(dt_config.dovetail_config['result_dir'], + file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], testcase+'.out') if not os.path.exists(file_path): self.logger.info('result file not found: %s' % file_path) diff --git a/dovetail/run.py b/dovetail/run.py index 865b3996..0c57b4ed 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -10,18 +10,19 @@ import click import sys +import os +import time import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils from parser import Parser from container import Container from testcase import Testcase from testcase import Testsuite from report import Report -from report import FunctestCrawler -from report import YardstickCrawler -from report import FunctestChecker -from report import YardstickChecker +from report import FunctestCrawler, YardstickCrawler +from report import FunctestChecker, YardstickChecker from conf.dovetail_config import DovetailConfig as dt_config @@ -48,6 +49,7 @@ def run_test(testsuite, testarea, logger): if value is not None and (testarea == 'full' or testarea in value): testarea_list.append(value) + duration = 0 for testcase_name in testarea_list: logger.info('>>[testcase]: %s' % (testcase_name)) testcase = Testcase.get(testcase_name) @@ -78,8 +80,11 @@ def run_test(testsuite, testarea, logger): if not testcase.prepare_cmd(): logger.error('failed to prepare testcase:%s' % testcase.name()) else: + start_time = time.time() for cmd in testcase.cmds: Container.exec_cmd(container_id, cmd) + end_time = time.time() + duration = end_time - start_time # testcase.post_condition() @@ -88,6 +93,8 @@ def run_test(testsuite, testarea, logger): db_result = Report.get_result(testcase) Report.check_result(testcase, db_result) + return duration + def validate_options(input_dict, logger): # for 'tag' option @@ -120,8 +127,20 @@ def create_logs(): Testsuite.create_log() +def clean_results_dir(): + result_path = dt_config.dovetail_config['result_dir'] + if os.path.exists(result_path): + if os.path.isdir(result_path): + cmd = 'sudo rm -rf %s/*' % (result_path) + dt_utils.exec_cmd(cmd, exit_on_error=False) + else: + print "result_dir in dovetail_config.yml is not a directory." + sys.exit(-1) + + def main(*args, **kwargs): """Dovetail compliance test entry!""" + clean_results_dir() create_logs() logger = dt_logger.Logger('run').getLogger() logger.info('================================================') @@ -138,8 +157,6 @@ def main(*args, **kwargs): if 'tag' in kwargs and kwargs['tag'] is not None: set_container_tags(kwargs['tag']) - load_testcase() - testsuite_yaml = load_testsuite(kwargs['testsuite']) testarea = kwargs['testarea'] testsuite_validation = False testarea_validation = False @@ -148,8 +165,10 @@ def main(*args, **kwargs): if kwargs['testsuite'] in dt_config.testsuite_supported: testsuite_validation = True if testsuite_validation and testarea_validation: - run_test(testsuite_yaml, testarea, logger) - Report.generate(testsuite_yaml, testarea) + testsuite_yaml = load_testsuite(kwargs['testsuite']) + load_testcase() + duration = run_test(testsuite_yaml, testarea, logger) + Report.generate(testsuite_yaml, testarea, duration) else: logger.error('invalid input commands, testsuite %s testarea %s' % (kwargs['testsuite'], testarea)) diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py index 5b003d1a..b4331ea1 100644 --- a/dovetail/tests/unit/test_parser.py +++ b/dovetail/tests/unit/test_parser.py @@ -17,7 +17,7 @@ import unittest import yaml -import dovetail.parser as dovetail_parser +import parser as dovetail_parser class TestParser(unittest.TestCase): @@ -27,6 +27,7 @@ class TestParser(unittest.TestCase): def setUp(self): """Test case setup""" logging.disable(logging.CRITICAL) + dovetail_parser.Parser.create_log() def test_parser_cmd(self): """Test whether the command is correctly parsed.""" diff --git a/dovetail/utils/dovetail_logger.py b/dovetail/utils/dovetail_logger.py index e725e612..8afa08a1 100644 --- a/dovetail/utils/dovetail_logger.py +++ b/dovetail/utils/dovetail_logger.py @@ -25,16 +25,6 @@ import logging import os from conf.dovetail_config import DovetailConfig as dt_config -import dovetail_utils as dt_utils - - -def clean_results_dir(): - result_path = dt_config.dovetail_config['result_dir'] - if os.path.exists(result_path): - cmd = 'sudo rm -rf %s/*' % (result_path) - dt_utils.exec_cmd(cmd, exit_on_error=False) - -clean_results_dir() class Logger: diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index 2e33b53a..e4f91987 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -38,7 +38,7 @@ def exec_cmd(cmd, logger=None, exit_on_error=True, info=False, else: logger.debug(line) else: - print line + print (line) sys.stdout.flush() returncode = p.returncode diff --git a/ez_setup.py b/ez_setup.py index 4ef3ee01..36da1e62 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -125,7 +125,7 @@ def _do_download(version, download_base, to_dir, download_delay): egg = os.path.join(to_dir, tp.format(**locals())) if not os.path.exists(egg): archive = download_setuptools(version, download_base, - to_dir, download_delay) + to_dir, download_delay) _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) @@ -235,7 +235,8 @@ def download_file_powershell(url, target): ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " - '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' + '(new-object System.Net.WebClient).DownloadFile("%(url)s", \ + "%(target)s")' % locals() ) cmd = [ |