summaryrefslogtreecommitdiffstats
path: root/dovetail
diff options
context:
space:
mode:
authorxudan <xudan16@huawei.com>2018-11-24 01:44:02 -0500
committerxudan <xudan16@huawei.com>2019-01-03 22:15:40 -0500
commitb7ee5abe9b73cbc0928bf9ac080a6aecf05bcb7e (patch)
tree6825cc66028c7c085364a5cedd727cddc93b5b49 /dovetail
parent78b0062ce9dc4d23b967112a0896f12cc6526e1c (diff)
Add ONAP VNF SDK verification tests
Please refer to env_config.sh.onap.sample when testing it. The guide of preparing local env for onap package validation can be found here https://gerrit.onap.org/r/#/c/75119 Change-Id: I39d6b11544847756126623a7eb3b953dc800c470 Signed-off-by: xudan <xudan16@huawei.com>
Diffstat (limited to 'dovetail')
-rw-r--r--dovetail/report.py82
-rwxr-xr-xdovetail/run.py2
-rw-r--r--dovetail/test_runner.py21
-rw-r--r--dovetail/testcase.py12
-rw-r--r--dovetail/tests/unit/test_report.py204
-rw-r--r--dovetail/tests/unit/test_test_runner.py37
-rw-r--r--dovetail/tests/unit/test_testcase.py2
-rw-r--r--dovetail/utils/dovetail_utils.py2
8 files changed, 352 insertions, 10 deletions
diff --git a/dovetail/report.py b/dovetail/report.py
index f11bb24..ce88ed1 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -29,7 +29,7 @@ from testcase import Testcase
class Report(object):
results = {'functest': {}, 'yardstick': {}, 'functest-k8s': {},
- 'bottlenecks': {}, 'shell': {}, 'vnftest': {}}
+ 'bottlenecks': {}, 'shell': {}, 'vnftest': {}, 'onap-vtp': {}}
logger = None
@@ -431,6 +431,63 @@ class VnftestCrawler(Crawler):
return json_results
+class OnapVtpCrawler(Crawler):
+
+ logger = None
+
+ def __init__(self):
+ self.type = 'onap-vtp'
+ self.logger.debug('Create crawler: {}'.format(self.type))
+
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__ + '.OnapVtpCrawler').getLogger()
+
+ def crawl(self, testcase, file_path):
+ return self.crawl_from_file(testcase, file_path)
+
+ # The pass result looks like
+ # {
+ # "results": [
+ # {"property": "results", "value": "{value=SUCCESS}"},
+ # {"property": "build_tag", "value": "test"},
+ # {"property": "criteria", "value": "PASS"}
+ # ]
+ # }
+ # The fail result looks like
+ # {
+ # "results": [
+ # {"property": "results", "value": "{value=file doesn't exists}"},
+ # {"property": "build_tag", "value": "test"},
+ # {"property": "criteria", "value": "FAILED"}
+ # ]
+ # }
+ def crawl_from_file(self, testcase, file_path):
+ if not os.path.exists(file_path):
+ self.logger.error('Result file not found: {}'.format(file_path))
+ return None
+ criteria = 'FAIL'
+ with open(file_path, 'r') as f:
+ for jsonfile in f:
+ try:
+ data = json.loads(jsonfile)
+ for item in data['results']:
+ if 'criteria' == item['property']:
+ if 'PASS' == item['value']:
+ criteria = 'PASS'
+ break
+ else:
+ self.logger.error('There is no property criteria.')
+ except KeyError as e:
+ self.logger.exception('Pass flag not found {}'.format(e))
+ except ValueError:
+ continue
+ json_results = {'criteria': criteria}
+
+ testcase.set_results(json_results)
+ return json_results
+
+
class CrawlerFactory(object):
CRAWLER_MAP = {'functest': FunctestCrawler,
@@ -438,7 +495,8 @@ class CrawlerFactory(object):
'bottlenecks': BottlenecksCrawler,
'vnftest': VnftestCrawler,
'shell': ShellCrawler,
- 'functest-k8s': FunctestK8sCrawler}
+ 'functest-k8s': FunctestK8sCrawler,
+ 'onap-vtp': OnapVtpCrawler}
@classmethod
def create(cls, type):
@@ -592,6 +650,23 @@ class VnftestChecker(object):
return
+class OnapVtpChecker(object):
+
+ logger = None
+
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__ + '.OnapVtpChecker').getLogger()
+
+ @staticmethod
+ def check(testcase, result):
+ if not result:
+ testcase.passed('FAIL')
+ else:
+ testcase.passed(result['criteria'])
+ return
+
+
class CheckerFactory(object):
CHECKER_MAP = {'functest': FunctestChecker,
@@ -599,7 +674,8 @@ class CheckerFactory(object):
'bottlenecks': BottlenecksChecker,
'shell': ShellChecker,
'vnftest': VnftestChecker,
- 'functest-k8s': FunctestK8sChecker}
+ 'functest-k8s': FunctestK8sChecker,
+ 'onap-vtp': OnapVtpChecker}
@classmethod
def create(cls, type):
diff --git a/dovetail/run.py b/dovetail/run.py
index 1579ff6..7bb707f 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -111,11 +111,13 @@ def create_logs():
dt_report.YardstickCrawler.create_log()
dt_report.VnftestCrawler.create_log()
dt_report.BottlenecksCrawler.create_log()
+ dt_report.OnapVtpCrawler.create_log()
dt_report.FunctestChecker.create_log()
dt_report.FunctestK8sChecker.create_log()
dt_report.YardstickChecker.create_log()
dt_report.VnftestChecker.create_log()
dt_report.BottlenecksChecker.create_log()
+ dt_report.OnapVtpChecker.create_log()
dt_testcase.Testcase.create_log()
dt_testcase.Testsuite.create_log()
dt_test_runner.DockerRunner.create_log()
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index 2fa1eee..228432a 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -130,6 +130,8 @@ class DockerRunner(Runner):
config_item['debug'] = os.getenv('DEBUG')
config_item['build_tag'] = dt_cfg.dovetail_config['build_tag']
config_item['cacert'] = os.getenv('OS_CACERT')
+ config_item['host_url'] = os.getenv('HOST_URL')
+ config_item['csar_file'] = os.getenv('CSAR_FILE')
return config_item
def _update_config(self, testcase, update_pod=True):
@@ -274,6 +276,22 @@ class VnftestRunner(DockerRunner):
super(VnftestRunner, self).__init__(testcase)
+class OnapVtpRunner(DockerRunner):
+
+ config_file_name = 'onap-vtp_config.yml'
+
+ def __init__(self, testcase):
+ self.type = 'onap-vtp'
+ super(OnapVtpRunner, self).__init__(testcase)
+ env_file = os.path.join(dt_cfg.dovetail_config['config_dir'],
+ dt_cfg.dovetail_config['env_file'])
+ if not os.path.isfile(env_file):
+ self.logger.error('File {} does not exist.'.format(env_file))
+ return
+ dt_utils.source_env(env_file)
+ self._update_config(testcase, update_pod=False)
+
+
class TestRunnerFactory(object):
TEST_RUNNER_MAP = {
@@ -282,7 +300,8 @@ class TestRunnerFactory(object):
"bottlenecks": BottlenecksRunner,
"shell": ShellRunner,
"vnftest": VnftestRunner,
- "functest-k8s": FunctestK8sRunner
+ "functest-k8s": FunctestK8sRunner,
+ "onap-vtp": OnapVtpRunner
}
@classmethod
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 74fbbea..04d95ca 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -349,6 +349,15 @@ class VnftestTestcase(Testcase):
self.type = 'vnftest'
+class OnapVtpTestcase(Testcase):
+
+ validate_testcase_list = {}
+
+ def __init__(self, testcase_yaml):
+ super(OnapVtpTestcase, self).__init__(testcase_yaml)
+ self.type = 'onap-vtp'
+
+
class TestcaseFactory(object):
TESTCASE_TYPE_MAP = {
'functest': FunctestTestcase,
@@ -356,7 +365,8 @@ class TestcaseFactory(object):
'bottlenecks': BottlenecksTestcase,
'shell': ShellTestcase,
'vnftest': VnftestTestcase,
- 'functest-k8s': FunctestK8sTestcase
+ 'functest-k8s': FunctestK8sTestcase,
+ 'onap-vtp': OnapVtpTestcase
}
@classmethod
diff --git a/dovetail/tests/unit/test_report.py b/dovetail/tests/unit/test_report.py
index 0f0d3ee..9f5369e 100644
--- a/dovetail/tests/unit/test_report.py
+++ b/dovetail/tests/unit/test_report.py
@@ -34,15 +34,17 @@ class ReportTesting(unittest.TestCase):
dt_report.YardstickCrawler.logger = None
dt_report.BottlenecksCrawler.logger = None
dt_report.VnftestCrawler.logger = None
+ dt_report.OnapVtpCrawler.logger = None
dt_report.FunctestChecker.logger = None
dt_report.FunctestK8sChecker.logger = None
dt_report.YardstickChecker.logger = None
dt_report.BottlenecksChecker.logger = None
dt_report.VnftestChecker.logger = None
+ dt_report.OnapVtpChecker.logger = None
dt_report.Report.logger = None
dt_report.Report.results = {
'functest': {}, 'yardstick': {}, 'functest-k8s': {},
- 'bottlenecks': {}, 'shell': {}, 'vnftest': {}}
+ 'bottlenecks': {}, 'shell': {}, 'vnftest': {}, 'onap-vtp': {}}
def _produce_report_initial_text(self, report_data):
report_txt = ''
@@ -947,6 +949,179 @@ class ReportTesting(unittest.TestCase):
"Pass flag not found 'result'")
self.assertEquals(expected, result)
+ @patch('dovetail.report.dt_logger')
+ def test_onapvtp_crawler_create_log(self, mock_logger):
+ getlogger_obj = Mock()
+ logger_obj = Mock()
+ logger_obj.getLogger.return_value = getlogger_obj
+ mock_logger.Logger.return_value = logger_obj
+
+ dt_report.OnapVtpCrawler.create_log()
+
+ self.assertEquals(getlogger_obj, dt_report.OnapVtpCrawler.logger)
+
+ @patch('dovetail.report.os.path')
+ def test_onapvtp_crawler_crawl_not_exists(self, mock_path):
+ logger_obj = Mock()
+ dt_report.OnapVtpCrawler.logger = logger_obj
+ mock_path.exists.return_value = False
+ file_path = 'file_path'
+
+ crawler = dt_report.OnapVtpCrawler()
+ result = crawler.crawl(None, file_path)
+
+ mock_path.exists.assert_called_once_with(file_path)
+ logger_obj.error.assert_called_once_with(
+ 'Result file not found: {}'.format(file_path))
+ self.assertEquals(None, result)
+
+ @patch('__builtin__.open')
+ @patch('dovetail.report.json.loads')
+ @patch('dovetail.report.os.path')
+ def test_onapvtp_crawler_crawl_pass(self, mock_path, mock_loads,
+ mock_open):
+ dt_report.OnapVtpCrawler.logger = Mock()
+ mock_path.exists.return_value = True
+ file_path = 'file_path'
+ testcase_obj = Mock()
+ file_obj = Mock()
+ mock_open.return_value.__enter__.return_value = [file_obj]
+ data_dict = {
+ 'results': [
+ {"property": "results", "value": "{value=SUCCESS}"},
+ {"property": "build_tag", "value": "test-name"},
+ {"property": "criteria", "value": "PASS"}
+ ]
+ }
+ mock_loads.return_value = data_dict
+
+ crawler = dt_report.OnapVtpCrawler()
+ result = crawler.crawl(testcase_obj, file_path)
+ expected = {'criteria': 'PASS'}
+
+ mock_path.exists.assert_called_once_with(file_path)
+ mock_open.assert_called_once_with(file_path, 'r')
+ mock_loads.assert_called_once_with(file_obj)
+ testcase_obj.set_results.assert_called_once_with(expected)
+ self.assertEquals(expected, result)
+
+ @patch('__builtin__.open')
+ @patch('dovetail.report.json.loads')
+ @patch('dovetail.report.os.path')
+ def test_onapvtp_crawler_crawl_fail(self, mock_path, mock_loads,
+ mock_open):
+ dt_report.OnapVtpCrawler.logger = Mock()
+ mock_path.exists.return_value = True
+ file_path = 'file_path'
+ testcase_obj = Mock()
+ file_obj = Mock()
+ mock_open.return_value.__enter__.return_value = [file_obj]
+ data_dict = {
+ 'results': [
+ {"property": "results", "value": "{value=file doesn't exist}"},
+ {"property": "build_tag", "value": "test-name"},
+ {"property": "criteria", "value": "FAILED"}
+ ]
+ }
+ mock_loads.return_value = data_dict
+
+ crawler = dt_report.OnapVtpCrawler()
+ result = crawler.crawl(testcase_obj, file_path)
+ expected = {'criteria': 'FAIL'}
+
+ mock_path.exists.assert_called_once_with(file_path)
+ mock_open.assert_called_once_with(file_path, 'r')
+ mock_loads.assert_called_once_with(file_obj)
+ testcase_obj.set_results.assert_called_once_with(expected)
+ self.assertEquals(expected, result)
+
+ @patch('__builtin__.open')
+ @patch('dovetail.report.json.loads')
+ @patch('dovetail.report.os.path')
+ def test_onapvtp_crawler_crawl_no_criteria(self, mock_path, mock_loads,
+ mock_open):
+ dt_report.OnapVtpCrawler.logger = Mock()
+ mock_path.exists.return_value = True
+ file_path = 'file_path'
+ testcase_obj = Mock()
+ file_obj = Mock()
+ mock_open.return_value.__enter__.return_value = [file_obj]
+ data_dict = {
+ 'results': [
+ {"property": "results", "value": "{value=file doesn't exist}"},
+ {"property": "build_tag", "value": "test-name"},
+ {"property": "error_criteria", "value": "FAILED"}
+ ]
+ }
+ mock_loads.return_value = data_dict
+
+ crawler = dt_report.OnapVtpCrawler()
+ result = crawler.crawl(testcase_obj, file_path)
+ expected = {'criteria': 'FAIL'}
+
+ mock_path.exists.assert_called_once_with(file_path)
+ mock_open.assert_called_once_with(file_path, 'r')
+ mock_loads.assert_called_once_with(file_obj)
+ dt_report.OnapVtpCrawler.logger.error.assert_called_once_with(
+ 'There is no property criteria.')
+ testcase_obj.set_results.assert_called_once_with(expected)
+ self.assertEquals(expected, result)
+
+ @patch('__builtin__.open')
+ @patch('dovetail.report.json.loads')
+ @patch('dovetail.report.os.path')
+ def test_onapvtp_crawler_crawl_exception(self, mock_path, mock_loads,
+ mock_open):
+ dt_report.OnapVtpCrawler.logger = Mock()
+ mock_path.exists.return_value = True
+ file_path = 'file_path'
+ testcase_obj = Mock()
+ file_obj = Mock()
+ mock_open.return_value.__enter__.return_value = [file_obj]
+ data_dict = {
+ 'error_results': [
+ {"property": "results", "value": "{value=file doesn't exist}"},
+ {"property": "build_tag", "value": "test-name"},
+ {"property": "error_criteria", "value": "FAILED"}
+ ]
+ }
+ mock_loads.return_value = data_dict
+
+ crawler = dt_report.OnapVtpCrawler()
+ result = crawler.crawl(testcase_obj, file_path)
+ expected = {'criteria': 'FAIL'}
+
+ mock_path.exists.assert_called_once_with(file_path)
+ mock_open.assert_called_once_with(file_path, 'r')
+ mock_loads.assert_called_once_with(file_obj)
+ dt_report.OnapVtpCrawler.logger.exception.assert_called_once_with(
+ "Pass flag not found 'results'")
+ testcase_obj.set_results.assert_called_once_with(expected)
+ self.assertEquals(expected, result)
+
+ @patch('__builtin__.open')
+ @patch('dovetail.report.json.loads')
+ @patch('dovetail.report.os.path')
+ def test_onapvtp_crawler_crawl_value_error(self, mock_path, mock_loads,
+ mock_open):
+ dt_report.OnapVtpCrawler.logger = Mock()
+ mock_path.exists.return_value = True
+ file_path = 'file_path'
+ testcase_obj = Mock()
+ file_obj = Mock()
+ mock_open.return_value.__enter__.return_value = [file_obj]
+ mock_loads.side_effect = ValueError('No JSON object could be decoded')
+
+ crawler = dt_report.OnapVtpCrawler()
+ result = crawler.crawl(testcase_obj, file_path)
+ expected = {'criteria': 'FAIL'}
+
+ mock_path.exists.assert_called_once_with(file_path)
+ mock_open.assert_called_once_with(file_path, 'r')
+ mock_loads.assert_called_once_with(file_obj)
+ testcase_obj.set_results.assert_called_once_with(expected)
+ self.assertEquals(expected, result)
+
def test_crawler_factory(self):
result = dt_report.CrawlerFactory.create('shell')
self.assertEquals(dt_report.ShellCrawler, result.__class__)
@@ -1157,3 +1332,30 @@ class ReportTesting(unittest.TestCase):
def test_checker_factory_none(self):
self.assertEquals(None, dt_report.CheckerFactory.create('other'))
+
+ @patch('dovetail.report.dt_logger')
+ def test_onapvtp_checker_create_log(self, mock_logger):
+ getlogger_obj = Mock()
+ logger_obj = Mock()
+ logger_obj.getLogger.return_value = getlogger_obj
+ mock_logger.Logger.return_value = logger_obj
+
+ dt_report.OnapVtpChecker.create_log()
+
+ self.assertEquals(getlogger_obj, dt_report.OnapVtpChecker.logger)
+
+ def test_onapvtp_check_result_none(self):
+ testcase_obj = Mock()
+ result = {}
+
+ dt_report.OnapVtpChecker.check(testcase_obj, result)
+
+ testcase_obj.passed.assert_called_once_with('FAIL')
+
+ def test_onapvtp_check_result(self):
+ testcase_obj = Mock()
+ result = {'criteria': 'PASS'}
+
+ dt_report.OnapVtpChecker.check(testcase_obj, result)
+
+ testcase_obj.passed.assert_called_once_with('PASS')
diff --git a/dovetail/tests/unit/test_test_runner.py b/dovetail/tests/unit/test_test_runner.py
index 2570ec7..4b5c00b 100644
--- a/dovetail/tests/unit/test_test_runner.py
+++ b/dovetail/tests/unit/test_test_runner.py
@@ -323,7 +323,7 @@ class TestRunnerTesting(unittest.TestCase):
@patch('dovetail.test_runner.os')
def test_add_testcase_info(self, mock_os, mock_config):
mock_os.getenv.side_effect = ['os_insecure', 'dovetail_home', 'debug',
- 'os_cacert']
+ 'os_cacert', 'host_url', 'csar_file']
mock_os.environ = {'DEPLOY_SCENARIO': 'deploy_scenario'}
mock_config.dovetail_config = {'build_tag': 'build_tag'}
@@ -332,7 +332,8 @@ class TestRunnerTesting(unittest.TestCase):
'testcase': 'testcase_name', 'os_insecure': 'os_insecure',
'deploy_scenario': 'deploy_scenario',
'dovetail_home': 'dovetail_home', 'debug': 'debug',
- 'build_tag': 'build_tag', 'cacert': 'os_cacert'}
+ 'build_tag': 'build_tag', 'cacert': 'os_cacert',
+ 'host_url': 'host_url', 'csar_file': 'csar_file'}
result = t_runner.FunctestRunner._add_testcase_info(self.testcase)
self.testcase.validate_testcase.assert_called_once_with()
@@ -639,3 +640,35 @@ class TestRunnerTesting(unittest.TestCase):
'pod_file': 'two',
'full_task_yaml': 'full_value'},
result)
+
+ @patch('dovetail.test_runner.dt_utils')
+ @patch('dovetail.test_runner.os.path')
+ @patch('dovetail.test_runner.dt_cfg')
+ def test_init_onapvtprunner_no_env_file(self, mock_config, mock_path,
+ mock_utils):
+ t_runner.OnapVtpRunner.create_log()
+ mock_path.join.side_effect = ['env_file']
+ mock_config.dovetail_config = {'config_dir': 'one', 'env_file': 'two'}
+ mock_path.isfile.return_value = False
+
+ docker_runner = t_runner.OnapVtpRunner(self.testcase)
+
+ mock_path.join.assert_has_calls([call('one', 'two')])
+ mock_path.isfile.assert_called_once()
+ docker_runner.logger.error.assert_called_once_with(
+ 'File env_file does not exist.')
+
+ @patch('dovetail.test_runner.dt_utils')
+ @patch('dovetail.test_runner.os.path')
+ @patch('dovetail.test_runner.dt_cfg')
+ def test_init_onapvtprunner(self, mock_config, mock_path, mock_utils):
+ t_runner.OnapVtpRunner.create_log()
+ mock_path.join.side_effect = ['env_file']
+ mock_config.dovetail_config = {'config_dir': 'one', 'env_file': 'two'}
+ mock_path.isfile.return_value = True
+
+ t_runner.OnapVtpRunner(self.testcase)
+
+ mock_path.join.assert_has_calls([call('one', 'two')])
+ mock_path.isfile.assert_called_once()
+ mock_utils.source_env.assert_called_once_with('env_file')
diff --git a/dovetail/tests/unit/test_testcase.py b/dovetail/tests/unit/test_testcase.py
index c3eb683..7224c1a 100644
--- a/dovetail/tests/unit/test_testcase.py
+++ b/dovetail/tests/unit/test_testcase.py
@@ -113,7 +113,7 @@ class TestcaseTesting(unittest.TestCase):
self.assertEquals(True, result)
def test_str(self):
- testcase = tcase.Testcase(self.testcase_yaml)
+ testcase = tcase.OnapVtpTestcase(self.testcase_yaml)
result = testcase.__str__()
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 2e7b610..0f26eb2 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -77,7 +77,7 @@ def exec_cmd(cmd, logger=None, exit_on_error=False, info=False,
# walkthrough the object, yield path and value
-# dual python 2/3 compatability, inspired by the "six" library
+# dual python 2/3 compatibility, inspired by the "six" library
string_types = (str, unicode) if str is bytes else (str, bytes)
# iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()