diff options
| -rw-r--r-- | dovetail/conf/functest_config.yml | 4 | ||||
| -rw-r--r-- | dovetail/container.py | 10 | ||||
| -rw-r--r-- | dovetail/report.py | 85 | ||||
| -rwxr-xr-x | dovetail/run.py | 4 | ||||
| -rw-r--r-- | dovetail/test_runner.py | 78 | ||||
| -rw-r--r-- | dovetail/testcase.py | 30 | ||||
| -rw-r--r-- | dovetail/testcase/example.tc002.yml | 1 | ||||
| -rw-r--r-- | dovetail/testcase/example.tc003.yml | 14 | ||||
| -rw-r--r-- | dovetail/testcase/ipv6.tc001.yml | 4 | ||||
| -rw-r--r-- | dovetail/utils/dovetail_utils.py | 2 |
10 files changed, 179 insertions, 53 deletions
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml index ceb894b3..6aea2809 100644 --- a/dovetail/conf/functest_config.yml +++ b/dovetail/conf/functest_config.yml @@ -6,12 +6,12 @@ functest: -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal' opts: '-id --privileged=true' pre_condition: - - 'echo test for precondition' + - 'echo test for precondition in functest' cmds: - 'functest env prepare' - 'functest testcase run {{validate_testcase}}' post_condition: - - 'echo test for postcondition' + - 'echo test for postcondition in functest' result: dir: '/home/opnfv/functest/results' store_type: 'file' diff --git a/dovetail/container.py b/dovetail/container.py index 59fc0d8a..b3450fa3 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -67,8 +67,10 @@ class Container: (docker_image)) else: cmd = 'sudo docker pull %s' % (docker_image) - dt_utils.exec_cmd(cmd, cls.logger) - cls.has_pull_latest_image[type] = True + ret, msg = dt_utils.exec_cmd(cmd, cls.logger) + if ret == 0: + cls.logger.debug('docker pull %s success!', docker_image) + cls.has_pull_latest_image[type] = True @classmethod def clean(cls, container_id): @@ -80,6 +82,6 @@ class Container: @classmethod def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): if sub_cmd == "": - return + return (1, 'sub_cmd is empty') cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd) - dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) + return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) diff --git a/dovetail/report.py b/dovetail/report.py index 654e6bf8..8c302b60 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -168,12 +168,13 @@ class Report: type = testcase.validate_type() crawler = CrawlerFactory.create(type) if crawler is None: + cls.logger.error('crawler is None:%s', testcase.name()) return None if validate_testcase in cls.results[type]: return cls.results[type][validate_testcase] - result = crawler.crawl(validate_testcase) + result = crawler.crawl(testcase) if result is not None: cls.results[type][validate_testcase] = result @@ -187,25 +188,13 @@ class Report: return result -class CrawlerFactory: - - @staticmethod - def create(type): - if type == 'functest': - return FunctestCrawler() - - if type == 'yardstick': - return YardstickCrawler() - - return None - - class FunctestCrawler: logger = None def __init__(self): self.type = 'functest' + self.logger.debug('create crawler:%s', self.type) @classmethod def create_log(cls): @@ -258,7 +247,8 @@ class FunctestCrawler: def crawl_from_url(self, testcase=None): url = \ - dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase + dt_cfg.dovetail_config[self.type]['result']['db_url'] % \ + testcase.validate_testcase() self.logger.debug("Query to rest api: %s" % url) try: data = json.load(urllib2.urlopen(url)) @@ -275,6 +265,7 @@ class YardstickCrawler: def __init__(self): self.type = 'yardstick' + self.logger.debug('create crawler:%s', self.type) @classmethod def create_log(cls): @@ -292,7 +283,7 @@ class YardstickCrawler: def crawl_from_file(self, testcase=None): file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], - testcase + '.out') + testcase.validate_testcase() + '.out') if not os.path.exists(file_path): self.logger.info('result file not found: %s' % file_path) return None @@ -312,17 +303,39 @@ class YardstickCrawler: return None -class CheckerFactory: +class ShellCrawler: - @staticmethod - def create(type): - if type == 'functest': - return FunctestChecker() + def __init__(self): + self.type = 'shell' - if type == 'yardstick': - return YardstickChecker() + def crawl(self, testcase=None): + return self.crawl_from_file(testcase) - return None + def crawl_from_file(self, testcase=None): + file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], + testcase.name()) + '.out' + if not os.path.exists(file_path): + return None + try: + with open(file_path, 'r') as json_data: + result = json.load(json_data) + return result + except Exception: + return None + + +class CrawlerFactory: + + CRAWLER_MAP = {'functest': FunctestCrawler, + 'yardstick': YardstickCrawler, + 'shell': ShellCrawler} + + @classmethod + def create(cls, type): + try: + return cls.CRAWLER_MAP[type]() + except KeyError: + return None class ResultChecker: @@ -388,3 +401,27 @@ class YardstickChecker: else: testcase.passed(result['criteria'] == 'PASS') return + + +class ShellChecker: + + @staticmethod + def check(testcase, result): + try: + testcase.passed(result['pass']) + except Exception: + testcase.passed(False) + + +class CheckerFactory: + + CHECKER_MAP = {'functest': FunctestChecker, + 'yardstick': YardstickChecker, + 'shell': ShellChecker} + + @classmethod + def create(cls, type): + try: + return cls.CHECKER_MAP[type]() + except KeyError: + return None diff --git a/dovetail/run.py b/dovetail/run.py index 8a91e4b3..85e65667 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -64,8 +64,8 @@ def run_test(testsuite, testarea, logger): end_time = time.time() duration = end_time - start_time - db_result = Report.get_result(testcase) - Report.check_result(testcase, db_result) + result = Report.get_result(testcase) + Report.check_result(testcase, result) return duration diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py index 8a95b1f7..70dd2345 100644 --- a/dovetail/test_runner.py +++ b/dovetail/test_runner.py @@ -7,8 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +import os +import json import utils.dovetail_utils as dt_utils import utils.dovetail_logger as dt_logger +from utils.dovetail_config import DovetailConfig as dt_cfg from container import Container @@ -19,34 +22,48 @@ class DockerRunner(object): def __init__(self, testcase): self.testcase = testcase + self.logger.debug('create runner: %s', self.type) @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__file__).getLogger() + cls.logger = dt_logger.Logger(__name__ + '.DockerRunner').getLogger() def run(self): Container.pull_image(self.testcase.validate_type()) container_id = Container.create(self.testcase.validate_type()) + if container_id == '': + self.logger.error('failed to create container') + return + self.logger.debug('container id:%s' % container_id) if not self.testcase.prepared(): + failed = False cmds = self.testcase.pre_condition() if cmds: for cmd in cmds: - Container.exec_cmd(container_id, cmd) - self.testcase.prepared(True) + ret, msg = Container.exec_cmd(container_id, cmd) + if ret != 0: + failed = True + break + if not failed: + self.testcase.prepared(True) if not self.testcase.prepare_cmd(): self.logger.error('failed to prepare testcase:%s', self.testcase.name) else: for cmd in self.testcase.cmds: - Container.exec_cmd(container_id, cmd) + ret, msg = Container.exec_cmd(container_id, cmd) + if ret != 0: + self.logger.error('Failed to exec %s, ret:%d, msg:%s', + cmd, ret, msg) + break cmds = self.testcase.post_condition() if cmds: for cmd in cmds: - Container.exec_cmd(container_id, cmd) + ret, msg = Container.exec_cmd(container_id, cmd) self.testcase.cleaned(True) Container.clean(container_id) @@ -55,15 +72,15 @@ class DockerRunner(object): class FunctestRunner(DockerRunner): def __init__(self, testcase): + self.type = 'functest' super(FunctestRunner, self).__init__(testcase) - self.name = 'functest' class YardstickRunner(DockerRunner): def __init__(self, testcase): + self.type = 'yardstick' super(YardstickRunner, self).__init__(testcase) - self.name = 'yardstick' class ShellRunner(object): @@ -72,16 +89,55 @@ class ShellRunner(object): @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__file__).getLogger() + cls.logger = dt_logger.Logger(__name__ + '.ShellRunner').getLogger() def __init__(self, testcase): super(ShellRunner, self).__init__() self.testcase = testcase - self.name = 'shell' + self.type = 'shell' + self.logger.debug('create runner:%s', self.type) def run(self): - for cmd in self.testcase.cmds: - dt_utils.exec_cmd(cmd, self.logger) + passed = True + failed = False + result = {'pass': True, 'results': []} + if not self.testcase.prepared(): + cmds = self.testcase.pre_condition() + for cmd in cmds: + ret, msg = dt_utils.exec_cmd(cmd, self.logger) + result['results'].append((cmd, ret, msg)) + if ret != 0: + failed = True + break + if not failed: + self.testcase.prepared(True) + + if not self.testcase.prepare_cmd(): + self.logger.error('failed to prepare cmd:%s', + self.testcase.name()) + else: + for cmd in self.testcase.cmds: + ret, msg = dt_utils.exec_cmd(cmd, self.logger) + result['results'].append((cmd, ret, msg)) + if ret != 0: + passed = False + + result['pass'] = passed + + cmds = self.testcase.post_condition() + for cmd in cmds: + ret, msg = dt_utils.exec_cmd(cmd, self.logger) + result['results'].append((cmd, ret, msg)) + + result_filename = os.path.join(dt_cfg.dovetail_config['result_dir'], + self.testcase.name()) + '.out' + self.logger.debug('save result:%s', result_filename) + try: + with open(result_filename, 'w') as f: + f.write(json.dumps(result)) + except Exception as e: + self.logger.exception('Failed to write result into file:%s, \ + except:%s', result_filename, e) class TestRunnerFactory(object): diff --git a/dovetail/testcase.py b/dovetail/testcase.py index dd0fd2b5..040c6f98 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -93,17 +93,33 @@ class Testcase(object): return self._result_acquired(self.validate_testcase(), acquired) def pre_condition(self): - return self.pre_condition_cls(self.validate_type()) + try: + pre_condition = self.testcase['validate']['pre_condition'] + if pre_condition == '': + pre_condition = self.pre_condition_cls(self.validate_type()) + return pre_condition + except: + self.logger.debug('testcase:%s pre_condition is empty', + self.name()) + return '' def post_condition(self): - return self.post_condition_cls(self.validate_type()) + try: + post_condition = self.testcase['validate']['post_condition'] + if post_condition == '': + post_condition = self.post_condition_cls(self.validate_type()) + return post_condition + except: + self.logger.debug('testcae:%s post_condition is empty', + self.name()) + return '' def run(self): runner = TestRunnerFactory.create(self) try: runner.run() - except AttributeError: - pass + except AttributeError as e: + self.logger.exception('testcase:%s except:%s', self.name, e) # testcase in upstream testing project # validate_testcase_list = {'functest': {}, 'yardstick': {}, 'shell': {}} @@ -187,7 +203,7 @@ class FunctestTestcase(Testcase): def __init__(self, testcase_yaml): super(FunctestTestcase, self).__init__(testcase_yaml) - self.name = 'functest' + self.type = 'functest' def prepare_cmd(self): ret = super(FunctestTestcase, self).prepare_cmd() @@ -208,7 +224,7 @@ class YardstickTestcase(Testcase): def __init__(self, testcase_yaml): super(YardstickTestcase, self).__init__(testcase_yaml) - self.name = 'yardstick' + self.type = 'yardstick' class ShellTestcase(Testcase): @@ -217,7 +233,7 @@ class ShellTestcase(Testcase): def __init__(self, testcase_yaml): super(ShellTestcase, self).__init__(testcase_yaml) - self.name = 'shell' + self.type = 'shell' class TestcaseFactory(object): diff --git a/dovetail/testcase/example.tc002.yml b/dovetail/testcase/example.tc002.yml index 89d000c9..b9296074 100644 --- a/dovetail/testcase/example.tc002.yml +++ b/dovetail/testcase/example.tc002.yml @@ -8,6 +8,7 @@ dovetail.example.tc002: - "echo pre_condition" cmds: - "echo test2" + - "mkdir xxx" post_condition: - "echo post_condition" report: diff --git a/dovetail/testcase/example.tc003.yml b/dovetail/testcase/example.tc003.yml new file mode 100644 index 00000000..7916e017 --- /dev/null +++ b/dovetail/testcase/example.tc003.yml @@ -0,0 +1,14 @@ +dovetail.example.tc003: + name: dovetail.example.tc003 + objective: doing something useful + validate: + type: shell + testcase: "run shell22" + pre_condition: + cmds: + - "echo test2" + post_condition: + - "" + report: + sub_testcase_list: + diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml index 0bc0baaa..916d0915 100644 --- a/dovetail/testcase/ipv6.tc001.yml +++ b/dovetail/testcase/ipv6.tc001.yml @@ -5,12 +5,12 @@ dovetail.ipv6.tc001: type: functest testcase: tempest_smoke_serial pre_condition: - - 'echo test for precondition' + - 'echo test for precondition in testcase' cmds: - 'functest env prepare' - 'functest testcase run {{validate_testcase}}' post_condition: - - 'echo test for precondition' + - 'echo test for precondition in testcase' report: sub_testcase_list: - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index 1c68b7f3..8c4865d8 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -34,7 +34,7 @@ def exec_log(verbose, logger, msg, level, flush=False): sys.stdout.flush() -def exec_cmd(cmd, logger=None, exit_on_error=True, info=False, +def exec_cmd(cmd, logger=None, exit_on_error=False, info=False, err_msg="", verbose=True): msg_err = ("The command '%s' failed." % cmd) if not err_msg else err_msg msg_exec = ("Executing command: '%s'" % cmd) |