summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--dovetail/conf/cmd_config.yml13
-rw-r--r--dovetail/conf/functest_config.yml4
-rw-r--r--dovetail/container.py10
-rw-r--r--dovetail/report.py85
-rwxr-xr-xdovetail/run.py4
-rw-r--r--dovetail/test_runner.py78
-rw-r--r--dovetail/testcase.py30
-rw-r--r--dovetail/testcase/example.tc002.yml1
-rw-r--r--dovetail/testcase/example.tc003.yml14
-rw-r--r--dovetail/testcase/ipv6.tc001.yml4
-rw-r--r--dovetail/utils/dovetail_config.py2
-rw-r--r--dovetail/utils/dovetail_utils.py2
12 files changed, 184 insertions, 63 deletions
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 1c6418cb..12c59354 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -30,14 +30,14 @@ cli:
- 'functest/envs'
- 'yardstick/envs'
help: 'IP of the system under test (SUT).'
- CON_DEBUG:
+ debug:
flags:
- - '--CON_DEBUG'
- - '-c'
+ - '--debug'
+ - '-d'
path:
- 'functest/envs'
- 'yardstick/envs'
- help: 'True for showing debug log in functest/yardstick container.'
+ help: 'True for showing debug log on screen.'
yard_tag:
flags:
- '--yard_tag'
@@ -69,8 +69,3 @@ cli:
- '--testarea'
default: 'full'
help: 'compliance testarea within testsuite'
- debug:
- flags:
- - '--debug'
- - '-d'
- help: 'True for showing debug log in screen.'
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index ceb894b3..6aea2809 100644
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -6,12 +6,12 @@ functest:
-e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal'
opts: '-id --privileged=true'
pre_condition:
- - 'echo test for precondition'
+ - 'echo test for precondition in functest'
cmds:
- 'functest env prepare'
- 'functest testcase run {{validate_testcase}}'
post_condition:
- - 'echo test for postcondition'
+ - 'echo test for postcondition in functest'
result:
dir: '/home/opnfv/functest/results'
store_type: 'file'
diff --git a/dovetail/container.py b/dovetail/container.py
index 59fc0d8a..b3450fa3 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -67,8 +67,10 @@ class Container:
(docker_image))
else:
cmd = 'sudo docker pull %s' % (docker_image)
- dt_utils.exec_cmd(cmd, cls.logger)
- cls.has_pull_latest_image[type] = True
+ ret, msg = dt_utils.exec_cmd(cmd, cls.logger)
+ if ret == 0:
+ cls.logger.debug('docker pull %s success!', docker_image)
+ cls.has_pull_latest_image[type] = True
@classmethod
def clean(cls, container_id):
@@ -80,6 +82,6 @@ class Container:
@classmethod
def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
if sub_cmd == "":
- return
+ return (1, 'sub_cmd is empty')
cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd)
- dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
+ return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
diff --git a/dovetail/report.py b/dovetail/report.py
index 654e6bf8..8c302b60 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -168,12 +168,13 @@ class Report:
type = testcase.validate_type()
crawler = CrawlerFactory.create(type)
if crawler is None:
+ cls.logger.error('crawler is None:%s', testcase.name())
return None
if validate_testcase in cls.results[type]:
return cls.results[type][validate_testcase]
- result = crawler.crawl(validate_testcase)
+ result = crawler.crawl(testcase)
if result is not None:
cls.results[type][validate_testcase] = result
@@ -187,25 +188,13 @@ class Report:
return result
-class CrawlerFactory:
-
- @staticmethod
- def create(type):
- if type == 'functest':
- return FunctestCrawler()
-
- if type == 'yardstick':
- return YardstickCrawler()
-
- return None
-
-
class FunctestCrawler:
logger = None
def __init__(self):
self.type = 'functest'
+ self.logger.debug('create crawler:%s', self.type)
@classmethod
def create_log(cls):
@@ -258,7 +247,8 @@ class FunctestCrawler:
def crawl_from_url(self, testcase=None):
url = \
- dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase
+ dt_cfg.dovetail_config[self.type]['result']['db_url'] % \
+ testcase.validate_testcase()
self.logger.debug("Query to rest api: %s" % url)
try:
data = json.load(urllib2.urlopen(url))
@@ -275,6 +265,7 @@ class YardstickCrawler:
def __init__(self):
self.type = 'yardstick'
+ self.logger.debug('create crawler:%s', self.type)
@classmethod
def create_log(cls):
@@ -292,7 +283,7 @@ class YardstickCrawler:
def crawl_from_file(self, testcase=None):
file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
- testcase + '.out')
+ testcase.validate_testcase() + '.out')
if not os.path.exists(file_path):
self.logger.info('result file not found: %s' % file_path)
return None
@@ -312,17 +303,39 @@ class YardstickCrawler:
return None
-class CheckerFactory:
+class ShellCrawler:
- @staticmethod
- def create(type):
- if type == 'functest':
- return FunctestChecker()
+ def __init__(self):
+ self.type = 'shell'
- if type == 'yardstick':
- return YardstickChecker()
+ def crawl(self, testcase=None):
+ return self.crawl_from_file(testcase)
- return None
+ def crawl_from_file(self, testcase=None):
+ file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
+ testcase.name()) + '.out'
+ if not os.path.exists(file_path):
+ return None
+ try:
+ with open(file_path, 'r') as json_data:
+ result = json.load(json_data)
+ return result
+ except Exception:
+ return None
+
+
+class CrawlerFactory:
+
+ CRAWLER_MAP = {'functest': FunctestCrawler,
+ 'yardstick': YardstickCrawler,
+ 'shell': ShellCrawler}
+
+ @classmethod
+ def create(cls, type):
+ try:
+ return cls.CRAWLER_MAP[type]()
+ except KeyError:
+ return None
class ResultChecker:
@@ -388,3 +401,27 @@ class YardstickChecker:
else:
testcase.passed(result['criteria'] == 'PASS')
return
+
+
+class ShellChecker:
+
+ @staticmethod
+ def check(testcase, result):
+ try:
+ testcase.passed(result['pass'])
+ except Exception:
+ testcase.passed(False)
+
+
+class CheckerFactory:
+
+ CHECKER_MAP = {'functest': FunctestChecker,
+ 'yardstick': YardstickChecker,
+ 'shell': ShellChecker}
+
+ @classmethod
+ def create(cls, type):
+ try:
+ return cls.CHECKER_MAP[type]()
+ except KeyError:
+ return None
diff --git a/dovetail/run.py b/dovetail/run.py
index 8a91e4b3..85e65667 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -64,8 +64,8 @@ def run_test(testsuite, testarea, logger):
end_time = time.time()
duration = end_time - start_time
- db_result = Report.get_result(testcase)
- Report.check_result(testcase, db_result)
+ result = Report.get_result(testcase)
+ Report.check_result(testcase, result)
return duration
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index 8a95b1f7..70dd2345 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -7,8 +7,11 @@
# http://www.apache.org/licenses/LICENSE-2.0
#
+import os
+import json
import utils.dovetail_utils as dt_utils
import utils.dovetail_logger as dt_logger
+from utils.dovetail_config import DovetailConfig as dt_cfg
from container import Container
@@ -19,34 +22,48 @@ class DockerRunner(object):
def __init__(self, testcase):
self.testcase = testcase
+ self.logger.debug('create runner: %s', self.type)
@classmethod
def create_log(cls):
- cls.logger = dt_logger.Logger(__file__).getLogger()
+ cls.logger = dt_logger.Logger(__name__ + '.DockerRunner').getLogger()
def run(self):
Container.pull_image(self.testcase.validate_type())
container_id = Container.create(self.testcase.validate_type())
+ if container_id == '':
+ self.logger.error('failed to create container')
+ return
+
self.logger.debug('container id:%s' % container_id)
if not self.testcase.prepared():
+ failed = False
cmds = self.testcase.pre_condition()
if cmds:
for cmd in cmds:
- Container.exec_cmd(container_id, cmd)
- self.testcase.prepared(True)
+ ret, msg = Container.exec_cmd(container_id, cmd)
+ if ret != 0:
+ failed = True
+ break
+ if not failed:
+ self.testcase.prepared(True)
if not self.testcase.prepare_cmd():
self.logger.error('failed to prepare testcase:%s',
self.testcase.name)
else:
for cmd in self.testcase.cmds:
- Container.exec_cmd(container_id, cmd)
+ ret, msg = Container.exec_cmd(container_id, cmd)
+ if ret != 0:
+ self.logger.error('Failed to exec %s, ret:%d, msg:%s',
+ cmd, ret, msg)
+ break
cmds = self.testcase.post_condition()
if cmds:
for cmd in cmds:
- Container.exec_cmd(container_id, cmd)
+ ret, msg = Container.exec_cmd(container_id, cmd)
self.testcase.cleaned(True)
Container.clean(container_id)
@@ -55,15 +72,15 @@ class DockerRunner(object):
class FunctestRunner(DockerRunner):
def __init__(self, testcase):
+ self.type = 'functest'
super(FunctestRunner, self).__init__(testcase)
- self.name = 'functest'
class YardstickRunner(DockerRunner):
def __init__(self, testcase):
+ self.type = 'yardstick'
super(YardstickRunner, self).__init__(testcase)
- self.name = 'yardstick'
class ShellRunner(object):
@@ -72,16 +89,55 @@ class ShellRunner(object):
@classmethod
def create_log(cls):
- cls.logger = dt_logger.Logger(__file__).getLogger()
+ cls.logger = dt_logger.Logger(__name__ + '.ShellRunner').getLogger()
def __init__(self, testcase):
super(ShellRunner, self).__init__()
self.testcase = testcase
- self.name = 'shell'
+ self.type = 'shell'
+ self.logger.debug('create runner:%s', self.type)
def run(self):
- for cmd in self.testcase.cmds:
- dt_utils.exec_cmd(cmd, self.logger)
+ passed = True
+ failed = False
+ result = {'pass': True, 'results': []}
+ if not self.testcase.prepared():
+ cmds = self.testcase.pre_condition()
+ for cmd in cmds:
+ ret, msg = dt_utils.exec_cmd(cmd, self.logger)
+ result['results'].append((cmd, ret, msg))
+ if ret != 0:
+ failed = True
+ break
+ if not failed:
+ self.testcase.prepared(True)
+
+ if not self.testcase.prepare_cmd():
+ self.logger.error('failed to prepare cmd:%s',
+ self.testcase.name())
+ else:
+ for cmd in self.testcase.cmds:
+ ret, msg = dt_utils.exec_cmd(cmd, self.logger)
+ result['results'].append((cmd, ret, msg))
+ if ret != 0:
+ passed = False
+
+ result['pass'] = passed
+
+ cmds = self.testcase.post_condition()
+ for cmd in cmds:
+ ret, msg = dt_utils.exec_cmd(cmd, self.logger)
+ result['results'].append((cmd, ret, msg))
+
+ result_filename = os.path.join(dt_cfg.dovetail_config['result_dir'],
+ self.testcase.name()) + '.out'
+ self.logger.debug('save result:%s', result_filename)
+ try:
+ with open(result_filename, 'w') as f:
+ f.write(json.dumps(result))
+ except Exception as e:
+ self.logger.exception('Failed to write result into file:%s, \
+ except:%s', result_filename, e)
class TestRunnerFactory(object):
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index dd0fd2b5..040c6f98 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -93,17 +93,33 @@ class Testcase(object):
return self._result_acquired(self.validate_testcase(), acquired)
def pre_condition(self):
- return self.pre_condition_cls(self.validate_type())
+ try:
+ pre_condition = self.testcase['validate']['pre_condition']
+ if pre_condition == '':
+ pre_condition = self.pre_condition_cls(self.validate_type())
+ return pre_condition
+ except:
+ self.logger.debug('testcase:%s pre_condition is empty',
+ self.name())
+ return ''
def post_condition(self):
- return self.post_condition_cls(self.validate_type())
+ try:
+ post_condition = self.testcase['validate']['post_condition']
+ if post_condition == '':
+ post_condition = self.post_condition_cls(self.validate_type())
+ return post_condition
+ except:
+ self.logger.debug('testcae:%s post_condition is empty',
+ self.name())
+ return ''
def run(self):
runner = TestRunnerFactory.create(self)
try:
runner.run()
- except AttributeError:
- pass
+ except AttributeError as e:
+ self.logger.exception('testcase:%s except:%s', self.name, e)
# testcase in upstream testing project
# validate_testcase_list = {'functest': {}, 'yardstick': {}, 'shell': {}}
@@ -187,7 +203,7 @@ class FunctestTestcase(Testcase):
def __init__(self, testcase_yaml):
super(FunctestTestcase, self).__init__(testcase_yaml)
- self.name = 'functest'
+ self.type = 'functest'
def prepare_cmd(self):
ret = super(FunctestTestcase, self).prepare_cmd()
@@ -208,7 +224,7 @@ class YardstickTestcase(Testcase):
def __init__(self, testcase_yaml):
super(YardstickTestcase, self).__init__(testcase_yaml)
- self.name = 'yardstick'
+ self.type = 'yardstick'
class ShellTestcase(Testcase):
@@ -217,7 +233,7 @@ class ShellTestcase(Testcase):
def __init__(self, testcase_yaml):
super(ShellTestcase, self).__init__(testcase_yaml)
- self.name = 'shell'
+ self.type = 'shell'
class TestcaseFactory(object):
diff --git a/dovetail/testcase/example.tc002.yml b/dovetail/testcase/example.tc002.yml
index 89d000c9..b9296074 100644
--- a/dovetail/testcase/example.tc002.yml
+++ b/dovetail/testcase/example.tc002.yml
@@ -8,6 +8,7 @@ dovetail.example.tc002:
- "echo pre_condition"
cmds:
- "echo test2"
+ - "mkdir xxx"
post_condition:
- "echo post_condition"
report:
diff --git a/dovetail/testcase/example.tc003.yml b/dovetail/testcase/example.tc003.yml
new file mode 100644
index 00000000..7916e017
--- /dev/null
+++ b/dovetail/testcase/example.tc003.yml
@@ -0,0 +1,14 @@
+dovetail.example.tc003:
+ name: dovetail.example.tc003
+ objective: doing something useful
+ validate:
+ type: shell
+ testcase: "run shell22"
+ pre_condition:
+ cmds:
+ - "echo test2"
+ post_condition:
+ - ""
+ report:
+ sub_testcase_list:
+
diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml
index 0bc0baaa..916d0915 100644
--- a/dovetail/testcase/ipv6.tc001.yml
+++ b/dovetail/testcase/ipv6.tc001.yml
@@ -5,12 +5,12 @@ dovetail.ipv6.tc001:
type: functest
testcase: tempest_smoke_serial
pre_condition:
- - 'echo test for precondition'
+ - 'echo test for precondition in testcase'
cmds:
- 'functest env prepare'
- 'functest testcase run {{validate_testcase}}'
post_condition:
- - 'echo test for precondition'
+ - 'echo test for precondition in testcase'
report:
sub_testcase_list:
- tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network
diff --git a/dovetail/utils/dovetail_config.py b/dovetail/utils/dovetail_config.py
index 33cf57be..7d2c1645 100644
--- a/dovetail/utils/dovetail_config.py
+++ b/dovetail/utils/dovetail_config.py
@@ -19,7 +19,7 @@ class DovetailConfig:
CMD_NAME_TRANS = {
'SUT_TYPE': 'INSTALLER_TYPE',
'SUT_IP': 'INSTALLER_IP',
- 'CON_DEBUG': 'CI_DEBUG',
+ 'DEBUG': 'CI_DEBUG',
}
@classmethod
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 1c68b7f3..8c4865d8 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -34,7 +34,7 @@ def exec_log(verbose, logger, msg, level, flush=False):
sys.stdout.flush()
-def exec_cmd(cmd, logger=None, exit_on_error=True, info=False,
+def exec_cmd(cmd, logger=None, exit_on_error=False, info=False,
err_msg="", verbose=True):
msg_err = ("The command '%s' failed." % cmd) if not err_msg else err_msg
msg_exec = ("Executing command: '%s'" % cmd)