From b7ee5abe9b73cbc0928bf9ac080a6aecf05bcb7e Mon Sep 17 00:00:00 2001 From: xudan Date: Sat, 24 Nov 2018 01:44:02 -0500 Subject: Add ONAP VNF SDK verification tests Please refer to env_config.sh.onap.sample when testing it. The guide of preparing local env for onap package validation can be found here https://gerrit.onap.org/r/#/c/75119 Change-Id: I39d6b11544847756126623a7eb3b953dc800c470 Signed-off-by: xudan --- dovetail/report.py | 82 ++++++++++++- dovetail/run.py | 2 + dovetail/test_runner.py | 21 +++- dovetail/testcase.py | 12 +- dovetail/tests/unit/test_report.py | 204 ++++++++++++++++++++++++++++++- dovetail/tests/unit/test_test_runner.py | 37 +++++- dovetail/tests/unit/test_testcase.py | 2 +- dovetail/utils/dovetail_utils.py | 2 +- etc/conf/onap-vtp_config.yml | 25 ++++ etc/testcase/onap-vtp.validate.csar.yml | 14 +++ etc/userconfig/env_config.sh.onap.sample | 4 + 11 files changed, 395 insertions(+), 10 deletions(-) create mode 100644 etc/conf/onap-vtp_config.yml create mode 100644 etc/testcase/onap-vtp.validate.csar.yml create mode 100644 etc/userconfig/env_config.sh.onap.sample diff --git a/dovetail/report.py b/dovetail/report.py index f11bb242..ce88ed13 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -29,7 +29,7 @@ from testcase import Testcase class Report(object): results = {'functest': {}, 'yardstick': {}, 'functest-k8s': {}, - 'bottlenecks': {}, 'shell': {}, 'vnftest': {}} + 'bottlenecks': {}, 'shell': {}, 'vnftest': {}, 'onap-vtp': {}} logger = None @@ -431,6 +431,63 @@ class VnftestCrawler(Crawler): return json_results +class OnapVtpCrawler(Crawler): + + logger = None + + def __init__(self): + self.type = 'onap-vtp' + self.logger.debug('Create crawler: {}'.format(self.type)) + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__ + '.OnapVtpCrawler').getLogger() + + def crawl(self, testcase, file_path): + return self.crawl_from_file(testcase, file_path) + + # The pass result looks like + # { + # "results": [ + # {"property": "results", "value": "{value=SUCCESS}"}, + # {"property": "build_tag", "value": "test"}, + # {"property": "criteria", "value": "PASS"} + # ] + # } + # The fail result looks like + # { + # "results": [ + # {"property": "results", "value": "{value=file doesn't exists}"}, + # {"property": "build_tag", "value": "test"}, + # {"property": "criteria", "value": "FAILED"} + # ] + # } + def crawl_from_file(self, testcase, file_path): + if not os.path.exists(file_path): + self.logger.error('Result file not found: {}'.format(file_path)) + return None + criteria = 'FAIL' + with open(file_path, 'r') as f: + for jsonfile in f: + try: + data = json.loads(jsonfile) + for item in data['results']: + if 'criteria' == item['property']: + if 'PASS' == item['value']: + criteria = 'PASS' + break + else: + self.logger.error('There is no property criteria.') + except KeyError as e: + self.logger.exception('Pass flag not found {}'.format(e)) + except ValueError: + continue + json_results = {'criteria': criteria} + + testcase.set_results(json_results) + return json_results + + class CrawlerFactory(object): CRAWLER_MAP = {'functest': FunctestCrawler, @@ -438,7 +495,8 @@ class CrawlerFactory(object): 'bottlenecks': BottlenecksCrawler, 'vnftest': VnftestCrawler, 'shell': ShellCrawler, - 'functest-k8s': FunctestK8sCrawler} + 'functest-k8s': FunctestK8sCrawler, + 'onap-vtp': OnapVtpCrawler} @classmethod def create(cls, type): @@ -592,6 +650,23 @@ class VnftestChecker(object): return +class OnapVtpChecker(object): + + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__ + '.OnapVtpChecker').getLogger() + + @staticmethod + def check(testcase, result): + if not result: + testcase.passed('FAIL') + else: + testcase.passed(result['criteria']) + return + + class CheckerFactory(object): CHECKER_MAP = {'functest': FunctestChecker, @@ -599,7 +674,8 @@ class CheckerFactory(object): 'bottlenecks': BottlenecksChecker, 'shell': ShellChecker, 'vnftest': VnftestChecker, - 'functest-k8s': FunctestK8sChecker} + 'functest-k8s': FunctestK8sChecker, + 'onap-vtp': OnapVtpChecker} @classmethod def create(cls, type): diff --git a/dovetail/run.py b/dovetail/run.py index 1579ff69..7bb707fd 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -111,11 +111,13 @@ def create_logs(): dt_report.YardstickCrawler.create_log() dt_report.VnftestCrawler.create_log() dt_report.BottlenecksCrawler.create_log() + dt_report.OnapVtpCrawler.create_log() dt_report.FunctestChecker.create_log() dt_report.FunctestK8sChecker.create_log() dt_report.YardstickChecker.create_log() dt_report.VnftestChecker.create_log() dt_report.BottlenecksChecker.create_log() + dt_report.OnapVtpChecker.create_log() dt_testcase.Testcase.create_log() dt_testcase.Testsuite.create_log() dt_test_runner.DockerRunner.create_log() diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py index 2fa1eee4..228432a5 100644 --- a/dovetail/test_runner.py +++ b/dovetail/test_runner.py @@ -130,6 +130,8 @@ class DockerRunner(Runner): config_item['debug'] = os.getenv('DEBUG') config_item['build_tag'] = dt_cfg.dovetail_config['build_tag'] config_item['cacert'] = os.getenv('OS_CACERT') + config_item['host_url'] = os.getenv('HOST_URL') + config_item['csar_file'] = os.getenv('CSAR_FILE') return config_item def _update_config(self, testcase, update_pod=True): @@ -274,6 +276,22 @@ class VnftestRunner(DockerRunner): super(VnftestRunner, self).__init__(testcase) +class OnapVtpRunner(DockerRunner): + + config_file_name = 'onap-vtp_config.yml' + + def __init__(self, testcase): + self.type = 'onap-vtp' + super(OnapVtpRunner, self).__init__(testcase) + env_file = os.path.join(dt_cfg.dovetail_config['config_dir'], + dt_cfg.dovetail_config['env_file']) + if not os.path.isfile(env_file): + self.logger.error('File {} does not exist.'.format(env_file)) + return + dt_utils.source_env(env_file) + self._update_config(testcase, update_pod=False) + + class TestRunnerFactory(object): TEST_RUNNER_MAP = { @@ -282,7 +300,8 @@ class TestRunnerFactory(object): "bottlenecks": BottlenecksRunner, "shell": ShellRunner, "vnftest": VnftestRunner, - "functest-k8s": FunctestK8sRunner + "functest-k8s": FunctestK8sRunner, + "onap-vtp": OnapVtpRunner } @classmethod diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 74fbbea8..04d95cae 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -349,6 +349,15 @@ class VnftestTestcase(Testcase): self.type = 'vnftest' +class OnapVtpTestcase(Testcase): + + validate_testcase_list = {} + + def __init__(self, testcase_yaml): + super(OnapVtpTestcase, self).__init__(testcase_yaml) + self.type = 'onap-vtp' + + class TestcaseFactory(object): TESTCASE_TYPE_MAP = { 'functest': FunctestTestcase, @@ -356,7 +365,8 @@ class TestcaseFactory(object): 'bottlenecks': BottlenecksTestcase, 'shell': ShellTestcase, 'vnftest': VnftestTestcase, - 'functest-k8s': FunctestK8sTestcase + 'functest-k8s': FunctestK8sTestcase, + 'onap-vtp': OnapVtpTestcase } @classmethod diff --git a/dovetail/tests/unit/test_report.py b/dovetail/tests/unit/test_report.py index 0f0d3eec..9f5369ec 100644 --- a/dovetail/tests/unit/test_report.py +++ b/dovetail/tests/unit/test_report.py @@ -34,15 +34,17 @@ class ReportTesting(unittest.TestCase): dt_report.YardstickCrawler.logger = None dt_report.BottlenecksCrawler.logger = None dt_report.VnftestCrawler.logger = None + dt_report.OnapVtpCrawler.logger = None dt_report.FunctestChecker.logger = None dt_report.FunctestK8sChecker.logger = None dt_report.YardstickChecker.logger = None dt_report.BottlenecksChecker.logger = None dt_report.VnftestChecker.logger = None + dt_report.OnapVtpChecker.logger = None dt_report.Report.logger = None dt_report.Report.results = { 'functest': {}, 'yardstick': {}, 'functest-k8s': {}, - 'bottlenecks': {}, 'shell': {}, 'vnftest': {}} + 'bottlenecks': {}, 'shell': {}, 'vnftest': {}, 'onap-vtp': {}} def _produce_report_initial_text(self, report_data): report_txt = '' @@ -947,6 +949,179 @@ class ReportTesting(unittest.TestCase): "Pass flag not found 'result'") self.assertEquals(expected, result) + @patch('dovetail.report.dt_logger') + def test_onapvtp_crawler_create_log(self, mock_logger): + getlogger_obj = Mock() + logger_obj = Mock() + logger_obj.getLogger.return_value = getlogger_obj + mock_logger.Logger.return_value = logger_obj + + dt_report.OnapVtpCrawler.create_log() + + self.assertEquals(getlogger_obj, dt_report.OnapVtpCrawler.logger) + + @patch('dovetail.report.os.path') + def test_onapvtp_crawler_crawl_not_exists(self, mock_path): + logger_obj = Mock() + dt_report.OnapVtpCrawler.logger = logger_obj + mock_path.exists.return_value = False + file_path = 'file_path' + + crawler = dt_report.OnapVtpCrawler() + result = crawler.crawl(None, file_path) + + mock_path.exists.assert_called_once_with(file_path) + logger_obj.error.assert_called_once_with( + 'Result file not found: {}'.format(file_path)) + self.assertEquals(None, result) + + @patch('__builtin__.open') + @patch('dovetail.report.json.loads') + @patch('dovetail.report.os.path') + def test_onapvtp_crawler_crawl_pass(self, mock_path, mock_loads, + mock_open): + dt_report.OnapVtpCrawler.logger = Mock() + mock_path.exists.return_value = True + file_path = 'file_path' + testcase_obj = Mock() + file_obj = Mock() + mock_open.return_value.__enter__.return_value = [file_obj] + data_dict = { + 'results': [ + {"property": "results", "value": "{value=SUCCESS}"}, + {"property": "build_tag", "value": "test-name"}, + {"property": "criteria", "value": "PASS"} + ] + } + mock_loads.return_value = data_dict + + crawler = dt_report.OnapVtpCrawler() + result = crawler.crawl(testcase_obj, file_path) + expected = {'criteria': 'PASS'} + + mock_path.exists.assert_called_once_with(file_path) + mock_open.assert_called_once_with(file_path, 'r') + mock_loads.assert_called_once_with(file_obj) + testcase_obj.set_results.assert_called_once_with(expected) + self.assertEquals(expected, result) + + @patch('__builtin__.open') + @patch('dovetail.report.json.loads') + @patch('dovetail.report.os.path') + def test_onapvtp_crawler_crawl_fail(self, mock_path, mock_loads, + mock_open): + dt_report.OnapVtpCrawler.logger = Mock() + mock_path.exists.return_value = True + file_path = 'file_path' + testcase_obj = Mock() + file_obj = Mock() + mock_open.return_value.__enter__.return_value = [file_obj] + data_dict = { + 'results': [ + {"property": "results", "value": "{value=file doesn't exist}"}, + {"property": "build_tag", "value": "test-name"}, + {"property": "criteria", "value": "FAILED"} + ] + } + mock_loads.return_value = data_dict + + crawler = dt_report.OnapVtpCrawler() + result = crawler.crawl(testcase_obj, file_path) + expected = {'criteria': 'FAIL'} + + mock_path.exists.assert_called_once_with(file_path) + mock_open.assert_called_once_with(file_path, 'r') + mock_loads.assert_called_once_with(file_obj) + testcase_obj.set_results.assert_called_once_with(expected) + self.assertEquals(expected, result) + + @patch('__builtin__.open') + @patch('dovetail.report.json.loads') + @patch('dovetail.report.os.path') + def test_onapvtp_crawler_crawl_no_criteria(self, mock_path, mock_loads, + mock_open): + dt_report.OnapVtpCrawler.logger = Mock() + mock_path.exists.return_value = True + file_path = 'file_path' + testcase_obj = Mock() + file_obj = Mock() + mock_open.return_value.__enter__.return_value = [file_obj] + data_dict = { + 'results': [ + {"property": "results", "value": "{value=file doesn't exist}"}, + {"property": "build_tag", "value": "test-name"}, + {"property": "error_criteria", "value": "FAILED"} + ] + } + mock_loads.return_value = data_dict + + crawler = dt_report.OnapVtpCrawler() + result = crawler.crawl(testcase_obj, file_path) + expected = {'criteria': 'FAIL'} + + mock_path.exists.assert_called_once_with(file_path) + mock_open.assert_called_once_with(file_path, 'r') + mock_loads.assert_called_once_with(file_obj) + dt_report.OnapVtpCrawler.logger.error.assert_called_once_with( + 'There is no property criteria.') + testcase_obj.set_results.assert_called_once_with(expected) + self.assertEquals(expected, result) + + @patch('__builtin__.open') + @patch('dovetail.report.json.loads') + @patch('dovetail.report.os.path') + def test_onapvtp_crawler_crawl_exception(self, mock_path, mock_loads, + mock_open): + dt_report.OnapVtpCrawler.logger = Mock() + mock_path.exists.return_value = True + file_path = 'file_path' + testcase_obj = Mock() + file_obj = Mock() + mock_open.return_value.__enter__.return_value = [file_obj] + data_dict = { + 'error_results': [ + {"property": "results", "value": "{value=file doesn't exist}"}, + {"property": "build_tag", "value": "test-name"}, + {"property": "error_criteria", "value": "FAILED"} + ] + } + mock_loads.return_value = data_dict + + crawler = dt_report.OnapVtpCrawler() + result = crawler.crawl(testcase_obj, file_path) + expected = {'criteria': 'FAIL'} + + mock_path.exists.assert_called_once_with(file_path) + mock_open.assert_called_once_with(file_path, 'r') + mock_loads.assert_called_once_with(file_obj) + dt_report.OnapVtpCrawler.logger.exception.assert_called_once_with( + "Pass flag not found 'results'") + testcase_obj.set_results.assert_called_once_with(expected) + self.assertEquals(expected, result) + + @patch('__builtin__.open') + @patch('dovetail.report.json.loads') + @patch('dovetail.report.os.path') + def test_onapvtp_crawler_crawl_value_error(self, mock_path, mock_loads, + mock_open): + dt_report.OnapVtpCrawler.logger = Mock() + mock_path.exists.return_value = True + file_path = 'file_path' + testcase_obj = Mock() + file_obj = Mock() + mock_open.return_value.__enter__.return_value = [file_obj] + mock_loads.side_effect = ValueError('No JSON object could be decoded') + + crawler = dt_report.OnapVtpCrawler() + result = crawler.crawl(testcase_obj, file_path) + expected = {'criteria': 'FAIL'} + + mock_path.exists.assert_called_once_with(file_path) + mock_open.assert_called_once_with(file_path, 'r') + mock_loads.assert_called_once_with(file_obj) + testcase_obj.set_results.assert_called_once_with(expected) + self.assertEquals(expected, result) + def test_crawler_factory(self): result = dt_report.CrawlerFactory.create('shell') self.assertEquals(dt_report.ShellCrawler, result.__class__) @@ -1157,3 +1332,30 @@ class ReportTesting(unittest.TestCase): def test_checker_factory_none(self): self.assertEquals(None, dt_report.CheckerFactory.create('other')) + + @patch('dovetail.report.dt_logger') + def test_onapvtp_checker_create_log(self, mock_logger): + getlogger_obj = Mock() + logger_obj = Mock() + logger_obj.getLogger.return_value = getlogger_obj + mock_logger.Logger.return_value = logger_obj + + dt_report.OnapVtpChecker.create_log() + + self.assertEquals(getlogger_obj, dt_report.OnapVtpChecker.logger) + + def test_onapvtp_check_result_none(self): + testcase_obj = Mock() + result = {} + + dt_report.OnapVtpChecker.check(testcase_obj, result) + + testcase_obj.passed.assert_called_once_with('FAIL') + + def test_onapvtp_check_result(self): + testcase_obj = Mock() + result = {'criteria': 'PASS'} + + dt_report.OnapVtpChecker.check(testcase_obj, result) + + testcase_obj.passed.assert_called_once_with('PASS') diff --git a/dovetail/tests/unit/test_test_runner.py b/dovetail/tests/unit/test_test_runner.py index 2570ec76..4b5c00bb 100644 --- a/dovetail/tests/unit/test_test_runner.py +++ b/dovetail/tests/unit/test_test_runner.py @@ -323,7 +323,7 @@ class TestRunnerTesting(unittest.TestCase): @patch('dovetail.test_runner.os') def test_add_testcase_info(self, mock_os, mock_config): mock_os.getenv.side_effect = ['os_insecure', 'dovetail_home', 'debug', - 'os_cacert'] + 'os_cacert', 'host_url', 'csar_file'] mock_os.environ = {'DEPLOY_SCENARIO': 'deploy_scenario'} mock_config.dovetail_config = {'build_tag': 'build_tag'} @@ -332,7 +332,8 @@ class TestRunnerTesting(unittest.TestCase): 'testcase': 'testcase_name', 'os_insecure': 'os_insecure', 'deploy_scenario': 'deploy_scenario', 'dovetail_home': 'dovetail_home', 'debug': 'debug', - 'build_tag': 'build_tag', 'cacert': 'os_cacert'} + 'build_tag': 'build_tag', 'cacert': 'os_cacert', + 'host_url': 'host_url', 'csar_file': 'csar_file'} result = t_runner.FunctestRunner._add_testcase_info(self.testcase) self.testcase.validate_testcase.assert_called_once_with() @@ -639,3 +640,35 @@ class TestRunnerTesting(unittest.TestCase): 'pod_file': 'two', 'full_task_yaml': 'full_value'}, result) + + @patch('dovetail.test_runner.dt_utils') + @patch('dovetail.test_runner.os.path') + @patch('dovetail.test_runner.dt_cfg') + def test_init_onapvtprunner_no_env_file(self, mock_config, mock_path, + mock_utils): + t_runner.OnapVtpRunner.create_log() + mock_path.join.side_effect = ['env_file'] + mock_config.dovetail_config = {'config_dir': 'one', 'env_file': 'two'} + mock_path.isfile.return_value = False + + docker_runner = t_runner.OnapVtpRunner(self.testcase) + + mock_path.join.assert_has_calls([call('one', 'two')]) + mock_path.isfile.assert_called_once() + docker_runner.logger.error.assert_called_once_with( + 'File env_file does not exist.') + + @patch('dovetail.test_runner.dt_utils') + @patch('dovetail.test_runner.os.path') + @patch('dovetail.test_runner.dt_cfg') + def test_init_onapvtprunner(self, mock_config, mock_path, mock_utils): + t_runner.OnapVtpRunner.create_log() + mock_path.join.side_effect = ['env_file'] + mock_config.dovetail_config = {'config_dir': 'one', 'env_file': 'two'} + mock_path.isfile.return_value = True + + t_runner.OnapVtpRunner(self.testcase) + + mock_path.join.assert_has_calls([call('one', 'two')]) + mock_path.isfile.assert_called_once() + mock_utils.source_env.assert_called_once_with('env_file') diff --git a/dovetail/tests/unit/test_testcase.py b/dovetail/tests/unit/test_testcase.py index c3eb683e..7224c1ae 100644 --- a/dovetail/tests/unit/test_testcase.py +++ b/dovetail/tests/unit/test_testcase.py @@ -113,7 +113,7 @@ class TestcaseTesting(unittest.TestCase): self.assertEquals(True, result) def test_str(self): - testcase = tcase.Testcase(self.testcase_yaml) + testcase = tcase.OnapVtpTestcase(self.testcase_yaml) result = testcase.__str__() diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index 2e7b6102..0f26eb26 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -77,7 +77,7 @@ def exec_cmd(cmd, logger=None, exit_on_error=False, info=False, # walkthrough the object, yield path and value -# dual python 2/3 compatability, inspired by the "six" library +# dual python 2/3 compatibility, inspired by the "six" library string_types = (str, unicode) if str is bytes else (str, bytes) # iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)() diff --git a/etc/conf/onap-vtp_config.yml b/etc/conf/onap-vtp_config.yml new file mode 100644 index 00000000..85944c3b --- /dev/null +++ b/etc/conf/onap-vtp_config.yml @@ -0,0 +1,25 @@ +--- + +{% set validate_testcase = validate_testcase or '' %} +{% set testcase = testcase or '' %} +{% set build_tag = build_tag or '' %} +{% set csar_file = csar_file or '' %} +{% set host_url = host_url or '' %} +{% set result_dir = '/tmp/onap' %} + +onap-vtp: + image_name: nexus3.onap.org:10001/onap/cli + docker_tag: 2.0.5 + opts: '-td ' + envs: '-e OPEN_CLI_MODE=daemon -e BUILD_TAG={{build_tag}}-{{testcase}} + -e OPEN_CLI_PRODUCT_IN_USE=onap-vtp' + volumes: + - '-v {{dovetail_home}}/pre_config/{{csar_file}}:/{{csar_file}}' + - '-v {{dovetail_home}}/results:{{result_dir}}' + pre_condition: + - 'echo this is pre_condition' + cmds: + - "oclip vnftest-run --name {{validate_testcase}} --param csar=/{{csar_file}} + --host-url {{host_url}} --format json > {{result_dir}}/{{testcase}}.out" + post_condition: + - 'echo this is post_condition' diff --git a/etc/testcase/onap-vtp.validate.csar.yml b/etc/testcase/onap-vtp.validate.csar.yml new file mode 100644 index 00000000..4a9d0ebb --- /dev/null +++ b/etc/testcase/onap-vtp.validate.csar.yml @@ -0,0 +1,14 @@ +--- +onap-vtp.validate.csar: + name: onap-vtp.validate.csar + objective: onap csar validation + validate: + type: onap-vtp + testcase: csar-validate + report: + source_archive_files: + - onap-vtp.validate.csar.out + dest_archive_files: + - onap-vtp_logs/onap-vtp.validate.csar.out + check_results_file: onap-vtp_logs/onap-vtp.validate.csar.out + sub_testcase_list: diff --git a/etc/userconfig/env_config.sh.onap.sample b/etc/userconfig/env_config.sh.onap.sample new file mode 100644 index 00000000..d983b469 --- /dev/null +++ b/etc/userconfig/env_config.sh.onap.sample @@ -0,0 +1,4 @@ +export HOST_URL="http://:8702" + +# Absolute path of CSAR file, and should be copied to vtp container. +export CSAR_FILE="/opt/test.csar" -- cgit 1.2.3-korg