summaryrefslogtreecommitdiffstats
path: root/dovetail
diff options
context:
space:
mode:
Diffstat (limited to 'dovetail')
-rw-r--r--dovetail/conf/functest_config.yml4
-rw-r--r--dovetail/container.py14
-rw-r--r--dovetail/report.py85
-rwxr-xr-xdovetail/run.py4
-rw-r--r--dovetail/test_runner.py78
-rw-r--r--dovetail/testcase.py30
-rw-r--r--dovetail/testcase/example.tc002.yml1
-rw-r--r--dovetail/testcase/example.tc003.yml14
-rw-r--r--dovetail/testcase/ipv6.tc001.yml4
-rw-r--r--dovetail/tests/unit/test_parser.py10
-rw-r--r--dovetail/utils/dovetail_utils.py69
11 files changed, 219 insertions, 94 deletions
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index ceb894b3..6aea2809 100644
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -6,12 +6,12 @@ functest:
-e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal'
opts: '-id --privileged=true'
pre_condition:
- - 'echo test for precondition'
+ - 'echo test for precondition in functest'
cmds:
- 'functest env prepare'
- 'functest testcase run {{validate_testcase}}'
post_condition:
- - 'echo test for postcondition'
+ - 'echo test for postcondition in functest'
result:
dir: '/home/opnfv/functest/results'
store_type: 'file'
diff --git a/dovetail/container.py b/dovetail/container.py
index 62070442..b3450fa3 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -20,10 +20,10 @@ class Container:
logger = None
- def __init__(cls):
+ def __init__(self):
pass
- def __str__(cls):
+ def __str__(self):
pass
@classmethod
@@ -67,8 +67,10 @@ class Container:
(docker_image))
else:
cmd = 'sudo docker pull %s' % (docker_image)
- dt_utils.exec_cmd(cmd, cls.logger)
- cls.has_pull_latest_image[type] = True
+ ret, msg = dt_utils.exec_cmd(cmd, cls.logger)
+ if ret == 0:
+ cls.logger.debug('docker pull %s success!', docker_image)
+ cls.has_pull_latest_image[type] = True
@classmethod
def clean(cls, container_id):
@@ -80,6 +82,6 @@ class Container:
@classmethod
def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
if sub_cmd == "":
- return
+ return (1, 'sub_cmd is empty')
cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd)
- dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
+ return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
diff --git a/dovetail/report.py b/dovetail/report.py
index 654e6bf8..8c302b60 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -168,12 +168,13 @@ class Report:
type = testcase.validate_type()
crawler = CrawlerFactory.create(type)
if crawler is None:
+ cls.logger.error('crawler is None:%s', testcase.name())
return None
if validate_testcase in cls.results[type]:
return cls.results[type][validate_testcase]
- result = crawler.crawl(validate_testcase)
+ result = crawler.crawl(testcase)
if result is not None:
cls.results[type][validate_testcase] = result
@@ -187,25 +188,13 @@ class Report:
return result
-class CrawlerFactory:
-
- @staticmethod
- def create(type):
- if type == 'functest':
- return FunctestCrawler()
-
- if type == 'yardstick':
- return YardstickCrawler()
-
- return None
-
-
class FunctestCrawler:
logger = None
def __init__(self):
self.type = 'functest'
+ self.logger.debug('create crawler:%s', self.type)
@classmethod
def create_log(cls):
@@ -258,7 +247,8 @@ class FunctestCrawler:
def crawl_from_url(self, testcase=None):
url = \
- dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase
+ dt_cfg.dovetail_config[self.type]['result']['db_url'] % \
+ testcase.validate_testcase()
self.logger.debug("Query to rest api: %s" % url)
try:
data = json.load(urllib2.urlopen(url))
@@ -275,6 +265,7 @@ class YardstickCrawler:
def __init__(self):
self.type = 'yardstick'
+ self.logger.debug('create crawler:%s', self.type)
@classmethod
def create_log(cls):
@@ -292,7 +283,7 @@ class YardstickCrawler:
def crawl_from_file(self, testcase=None):
file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
- testcase + '.out')
+ testcase.validate_testcase() + '.out')
if not os.path.exists(file_path):
self.logger.info('result file not found: %s' % file_path)
return None
@@ -312,17 +303,39 @@ class YardstickCrawler:
return None
-class CheckerFactory:
+class ShellCrawler:
- @staticmethod
- def create(type):
- if type == 'functest':
- return FunctestChecker()
+ def __init__(self):
+ self.type = 'shell'
- if type == 'yardstick':
- return YardstickChecker()
+ def crawl(self, testcase=None):
+ return self.crawl_from_file(testcase)
- return None
+ def crawl_from_file(self, testcase=None):
+ file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
+ testcase.name()) + '.out'
+ if not os.path.exists(file_path):
+ return None
+ try:
+ with open(file_path, 'r') as json_data:
+ result = json.load(json_data)
+ return result
+ except Exception:
+ return None
+
+
+class CrawlerFactory:
+
+ CRAWLER_MAP = {'functest': FunctestCrawler,
+ 'yardstick': YardstickCrawler,
+ 'shell': ShellCrawler}
+
+ @classmethod
+ def create(cls, type):
+ try:
+ return cls.CRAWLER_MAP[type]()
+ except KeyError:
+ return None
class ResultChecker:
@@ -388,3 +401,27 @@ class YardstickChecker:
else:
testcase.passed(result['criteria'] == 'PASS')
return
+
+
+class ShellChecker:
+
+ @staticmethod
+ def check(testcase, result):
+ try:
+ testcase.passed(result['pass'])
+ except Exception:
+ testcase.passed(False)
+
+
+class CheckerFactory:
+
+ CHECKER_MAP = {'functest': FunctestChecker,
+ 'yardstick': YardstickChecker,
+ 'shell': ShellChecker}
+
+ @classmethod
+ def create(cls, type):
+ try:
+ return cls.CHECKER_MAP[type]()
+ except KeyError:
+ return None
diff --git a/dovetail/run.py b/dovetail/run.py
index 8a91e4b3..85e65667 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -64,8 +64,8 @@ def run_test(testsuite, testarea, logger):
end_time = time.time()
duration = end_time - start_time
- db_result = Report.get_result(testcase)
- Report.check_result(testcase, db_result)
+ result = Report.get_result(testcase)
+ Report.check_result(testcase, result)
return duration
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index 8a95b1f7..70dd2345 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -7,8 +7,11 @@
# http://www.apache.org/licenses/LICENSE-2.0
#
+import os
+import json
import utils.dovetail_utils as dt_utils
import utils.dovetail_logger as dt_logger
+from utils.dovetail_config import DovetailConfig as dt_cfg
from container import Container
@@ -19,34 +22,48 @@ class DockerRunner(object):
def __init__(self, testcase):
self.testcase = testcase
+ self.logger.debug('create runner: %s', self.type)
@classmethod
def create_log(cls):
- cls.logger = dt_logger.Logger(__file__).getLogger()
+ cls.logger = dt_logger.Logger(__name__ + '.DockerRunner').getLogger()
def run(self):
Container.pull_image(self.testcase.validate_type())
container_id = Container.create(self.testcase.validate_type())
+ if container_id == '':
+ self.logger.error('failed to create container')
+ return
+
self.logger.debug('container id:%s' % container_id)
if not self.testcase.prepared():
+ failed = False
cmds = self.testcase.pre_condition()
if cmds:
for cmd in cmds:
- Container.exec_cmd(container_id, cmd)
- self.testcase.prepared(True)
+ ret, msg = Container.exec_cmd(container_id, cmd)
+ if ret != 0:
+ failed = True
+ break
+ if not failed:
+ self.testcase.prepared(True)
if not self.testcase.prepare_cmd():
self.logger.error('failed to prepare testcase:%s',
self.testcase.name)
else:
for cmd in self.testcase.cmds:
- Container.exec_cmd(container_id, cmd)
+ ret, msg = Container.exec_cmd(container_id, cmd)
+ if ret != 0:
+ self.logger.error('Failed to exec %s, ret:%d, msg:%s',
+ cmd, ret, msg)
+ break
cmds = self.testcase.post_condition()
if cmds:
for cmd in cmds:
- Container.exec_cmd(container_id, cmd)
+ ret, msg = Container.exec_cmd(container_id, cmd)
self.testcase.cleaned(True)
Container.clean(container_id)
@@ -55,15 +72,15 @@ class DockerRunner(object):
class FunctestRunner(DockerRunner):
def __init__(self, testcase):
+ self.type = 'functest'
super(FunctestRunner, self).__init__(testcase)
- self.name = 'functest'
class YardstickRunner(DockerRunner):
def __init__(self, testcase):
+ self.type = 'yardstick'
super(YardstickRunner, self).__init__(testcase)
- self.name = 'yardstick'
class ShellRunner(object):
@@ -72,16 +89,55 @@ class ShellRunner(object):
@classmethod
def create_log(cls):
- cls.logger = dt_logger.Logger(__file__).getLogger()
+ cls.logger = dt_logger.Logger(__name__ + '.ShellRunner').getLogger()
def __init__(self, testcase):
super(ShellRunner, self).__init__()
self.testcase = testcase
- self.name = 'shell'
+ self.type = 'shell'
+ self.logger.debug('create runner:%s', self.type)
def run(self):
- for cmd in self.testcase.cmds:
- dt_utils.exec_cmd(cmd, self.logger)
+ passed = True
+ failed = False
+ result = {'pass': True, 'results': []}
+ if not self.testcase.prepared():
+ cmds = self.testcase.pre_condition()
+ for cmd in cmds:
+ ret, msg = dt_utils.exec_cmd(cmd, self.logger)
+ result['results'].append((cmd, ret, msg))
+ if ret != 0:
+ failed = True
+ break
+ if not failed:
+ self.testcase.prepared(True)
+
+ if not self.testcase.prepare_cmd():
+ self.logger.error('failed to prepare cmd:%s',
+ self.testcase.name())
+ else:
+ for cmd in self.testcase.cmds:
+ ret, msg = dt_utils.exec_cmd(cmd, self.logger)
+ result['results'].append((cmd, ret, msg))
+ if ret != 0:
+ passed = False
+
+ result['pass'] = passed
+
+ cmds = self.testcase.post_condition()
+ for cmd in cmds:
+ ret, msg = dt_utils.exec_cmd(cmd, self.logger)
+ result['results'].append((cmd, ret, msg))
+
+ result_filename = os.path.join(dt_cfg.dovetail_config['result_dir'],
+ self.testcase.name()) + '.out'
+ self.logger.debug('save result:%s', result_filename)
+ try:
+ with open(result_filename, 'w') as f:
+ f.write(json.dumps(result))
+ except Exception as e:
+ self.logger.exception('Failed to write result into file:%s, \
+ except:%s', result_filename, e)
class TestRunnerFactory(object):
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index dd0fd2b5..040c6f98 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -93,17 +93,33 @@ class Testcase(object):
return self._result_acquired(self.validate_testcase(), acquired)
def pre_condition(self):
- return self.pre_condition_cls(self.validate_type())
+ try:
+ pre_condition = self.testcase['validate']['pre_condition']
+ if pre_condition == '':
+ pre_condition = self.pre_condition_cls(self.validate_type())
+ return pre_condition
+ except:
+ self.logger.debug('testcase:%s pre_condition is empty',
+ self.name())
+ return ''
def post_condition(self):
- return self.post_condition_cls(self.validate_type())
+ try:
+ post_condition = self.testcase['validate']['post_condition']
+ if post_condition == '':
+ post_condition = self.post_condition_cls(self.validate_type())
+ return post_condition
+ except:
+ self.logger.debug('testcae:%s post_condition is empty',
+ self.name())
+ return ''
def run(self):
runner = TestRunnerFactory.create(self)
try:
runner.run()
- except AttributeError:
- pass
+ except AttributeError as e:
+ self.logger.exception('testcase:%s except:%s', self.name, e)
# testcase in upstream testing project
# validate_testcase_list = {'functest': {}, 'yardstick': {}, 'shell': {}}
@@ -187,7 +203,7 @@ class FunctestTestcase(Testcase):
def __init__(self, testcase_yaml):
super(FunctestTestcase, self).__init__(testcase_yaml)
- self.name = 'functest'
+ self.type = 'functest'
def prepare_cmd(self):
ret = super(FunctestTestcase, self).prepare_cmd()
@@ -208,7 +224,7 @@ class YardstickTestcase(Testcase):
def __init__(self, testcase_yaml):
super(YardstickTestcase, self).__init__(testcase_yaml)
- self.name = 'yardstick'
+ self.type = 'yardstick'
class ShellTestcase(Testcase):
@@ -217,7 +233,7 @@ class ShellTestcase(Testcase):
def __init__(self, testcase_yaml):
super(ShellTestcase, self).__init__(testcase_yaml)
- self.name = 'shell'
+ self.type = 'shell'
class TestcaseFactory(object):
diff --git a/dovetail/testcase/example.tc002.yml b/dovetail/testcase/example.tc002.yml
index 89d000c9..b9296074 100644
--- a/dovetail/testcase/example.tc002.yml
+++ b/dovetail/testcase/example.tc002.yml
@@ -8,6 +8,7 @@ dovetail.example.tc002:
- "echo pre_condition"
cmds:
- "echo test2"
+ - "mkdir xxx"
post_condition:
- "echo post_condition"
report:
diff --git a/dovetail/testcase/example.tc003.yml b/dovetail/testcase/example.tc003.yml
new file mode 100644
index 00000000..7916e017
--- /dev/null
+++ b/dovetail/testcase/example.tc003.yml
@@ -0,0 +1,14 @@
+dovetail.example.tc003:
+ name: dovetail.example.tc003
+ objective: doing something useful
+ validate:
+ type: shell
+ testcase: "run shell22"
+ pre_condition:
+ cmds:
+ - "echo test2"
+ post_condition:
+ - ""
+ report:
+ sub_testcase_list:
+
diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml
index 0bc0baaa..916d0915 100644
--- a/dovetail/testcase/ipv6.tc001.yml
+++ b/dovetail/testcase/ipv6.tc001.yml
@@ -5,12 +5,12 @@ dovetail.ipv6.tc001:
type: functest
testcase: tempest_smoke_serial
pre_condition:
- - 'echo test for precondition'
+ - 'echo test for precondition in testcase'
cmds:
- 'functest env prepare'
- 'functest testcase run {{validate_testcase}}'
post_condition:
- - 'echo test for precondition'
+ - 'echo test for precondition in testcase'
report:
sub_testcase_list:
- tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network
diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py
index 73ed2b72..5fb4dbb9 100644
--- a/dovetail/tests/unit/test_parser.py
+++ b/dovetail/tests/unit/test_parser.py
@@ -16,7 +16,8 @@ import os
import unittest
import yaml
-import parser as dovetail_parser
+from dovetail import parser
+from dovetail.utils.dovetail_config import DovetailConfig as dt_cfg
class TestParser(unittest.TestCase):
@@ -25,9 +26,8 @@ class TestParser(unittest.TestCase):
def setUp(self):
"""Test case setup"""
- from utils.dovetail_config import DovetailConfig as dt_cfg
dt_cfg.load_config_files()
- dovetail_parser.Parser.create_log()
+ parser.Parser.create_log()
logging.disable(logging.CRITICAL)
def test_parser_cmd(self):
@@ -39,7 +39,7 @@ class TestParser(unittest.TestCase):
MockTestcase = type('Testcase', (object,), {})
mock_testcase = MockTestcase()
mock_testcase.testcase = mock_testcase_yaml.values()[0]
- output = dovetail_parser.Parser.parse_cmd(mock_cmd, mock_testcase)
+ output = parser.Parser.parse_cmd(mock_cmd, mock_testcase)
expected_output = ("python /functest/ci/run_tests.py -t "
"tempest_smoke_serial -r")
self.assertEqual(expected_output, output)
@@ -52,7 +52,7 @@ class TestParser(unittest.TestCase):
MockTestcase = type('Testcase', (object,), {})
mock_testcase = MockTestcase()
mock_testcase.testcase = mock_testcase_yaml.values()
- output = dovetail_parser.Parser.parse_cmd(mock_cmd, mock_testcase)
+ output = parser.Parser.parse_cmd(mock_cmd, mock_testcase)
expected_output = ("python /functest/ci/run_tests.py -t "
"None -r")
self.assertEqual(expected_output, output)
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 2d5f0dd1..8c4865d8 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -15,23 +15,34 @@ import subprocess
from collections import Mapping, Set, Sequence
-def exec_cmd(cmd, logger=None, exit_on_error=True, info=False,
- error_msg="", verbose=True):
- if not error_msg:
- error_msg = ("The command '%s' failed." % cmd)
- msg_exec = ("Executing command: '%s'" % cmd)
- if verbose:
- if logger:
- if info:
- logger.info(msg_exec)
- else:
- logger.debug(msg_exec)
+def exec_log(verbose, logger, msg, level, flush=False):
+ if not verbose:
+ return
+
+ if logger:
+ if level == 'info':
+ logger.info(msg)
+ elif level == 'error':
+ logger.error(msg)
+ elif level == 'debug':
+ logger.debug(msg)
else:
- print(msg_exec)
- p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
+ pass
+ else:
+ print(msg)
+ if flush:
+ sys.stdout.flush()
+
+
+def exec_cmd(cmd, logger=None, exit_on_error=False, info=False,
+ err_msg="", verbose=True):
+ msg_err = ("The command '%s' failed." % cmd) if not err_msg else err_msg
+ msg_exec = ("Executing command: '%s'" % cmd)
+ level = 'info' if info else 'debug'
+ exec_log(verbose, logger, msg_exec, level)
- # show progress bar
+ p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
seconds = 0
while p.poll() is None:
seconds += 1
@@ -39,29 +50,17 @@ def exec_cmd(cmd, logger=None, exit_on_error=True, info=False,
show_progress_bar(seconds)
time.sleep(1)
- output = p.communicate()
- for line in output[0].strip().split('\n'):
- line = line.replace('\n', '')
- if logger:
- if info:
- logger.info(line)
- else:
- logger.debug(line)
- else:
- print (line)
- sys.stdout.flush()
-
- returncode = p.returncode
- if returncode != 0:
- if verbose:
- if logger:
- logger.error(error_msg)
- else:
- print(error_msg)
+ (stdout, stderr) = p.communicate()
+ if p.returncode == 0:
+ for line in stdout.strip().splitlines():
+ exec_log(verbose, logger, line, level, True)
+ else:
+ exec_log(verbose, logger, stderr, 'error')
+ exec_log(verbose, logger, msg_err, 'error')
if exit_on_error:
sys.exit(1)
- return returncode, output[0].strip()
+ return p.returncode, stdout.strip()
# walkthrough the object, yield path and value