From f14c32d24b28f99846767f288843bf36f222cfd1 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 12 Apr 2017 10:44:49 -0600 Subject: Renamed test application and added new command line arguments. JIRA: SNAPS-10 Change-Id: I809a93f67ee0931df9601a0095775fe6303bbfbb Signed-off-by: spisarski --- docs/how-to-use/Testing.rst | 11 ++- snaps/test_runner.py | 170 ++++++++++++++++++++++++++++++++++++++++++++ snaps/unit_test_suite.py | 143 ------------------------------------- 3 files changed, 178 insertions(+), 146 deletions(-) create mode 100644 snaps/test_runner.py delete mode 100644 snaps/unit_test_suite.py diff --git a/docs/how-to-use/Testing.rst b/docs/how-to-use/Testing.rst index 66b8e5d..af66c28 100644 --- a/docs/how-to-use/Testing.rst +++ b/docs/how-to-use/Testing.rst @@ -7,7 +7,7 @@ Execute the tests :: cd - python snaps/unit_test_suite.py -e -n + python snaps/test_runner.py -e -n | \* All Supported Arguments | \* -e [required - The path to the OpenStack RC file] @@ -16,8 +16,13 @@ Execute the tests | \* -p [optional - the proxy settings if required. Format : | \* -s [optional - the proxy command used for SSH connections] | \* -l [(default INFO) The log level] +| \* -u [optional - When set, the unit tests will be executed] +| \* -c [optional - When set, the connection tests will be executed] +| \* -a [optional - When set, the API tests will be executed] +| \* -i [optional - When set, the integration tests will be executed] | \* -k [optional - When set, tests project and user creation. Use only if host running tests has access to the cloud's admin network] -| \* -f [optional - When set, will not execute tests requiring Floating +| \* -f [optional - When set, will execute tests requiring Floating IPS] -| \* -u [optional - When set, the unit tests will be executed] +| \* -fm [optional - JSON string containing a dict() for flavor metadata default='{\"hw:mem_page_size\": \"any\"}'] +| \* -r [optional with default value of '1' - The number of test iterations to execute] diff --git a/snaps/test_runner.py b/snaps/test_runner.py new file mode 100644 index 0000000..a8f2313 --- /dev/null +++ b/snaps/test_runner.py @@ -0,0 +1,170 @@ +# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") +# and others. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import logging +import os +import unittest + +from snaps import test_suite_builder + +__author__ = 'spisarski' + +logger = logging.getLogger('test_runner') + +ARG_NOT_SET = "argument not set" +LOG_LEVELS = {'FATAL': logging.FATAL, 'CRITICAL': logging.CRITICAL, 'ERROR': logging.ERROR, 'WARN': logging.WARN, + 'INFO': logging.INFO, 'DEBUG': logging.DEBUG} + + +def __create_test_suite(source_filename, ext_net_name, proxy_settings, ssh_proxy_cmd, run_unit_tests, + run_connection_tests, run_api_tests, run_integration_tests, flavor_metadata, use_keystone, + use_floating_ips, log_level): + """ + Compiles the tests that should run + :param source_filename: the OpenStack credentials file (required) + :param ext_net_name: the name of the external network to use for floating IPs (required) + :param run_unit_tests: when true, the tests not requiring OpenStack will be added to the test suite + :param run_connection_tests: when true, the tests that perform simple connections to OpenStack are executed + :param run_api_tests: when true, the tests that perform simple API calls to OpenStack are executed + :param run_integration_tests: when true, the integration tests are executed + :param proxy_settings: : of the proxy server (optional) + :param ssh_proxy_cmd: the command used to connect via SSH over some proxy server (optional) + :param flavor_metadata: dict() object containing the metadata for flavors created for test VM instance + :param use_keystone: when true, tests creating users and projects will be exercised and must be run on a host that + has access to the cloud's administrative network + :param use_floating_ips: when true, tests requiring floating IPs will be executed + :param log_level: the logging level + :return: + """ + suite = unittest.TestSuite() + + # Tests that do not require a remote connection to an OpenStack cloud + if run_unit_tests: + test_suite_builder.add_unit_tests(suite) + + # Basic connection tests + if run_connection_tests: + test_suite_builder.add_openstack_client_tests(suite, source_filename, ext_net_name, use_keystone=use_keystone, + http_proxy_str=proxy_settings, log_level=log_level) + + # Tests the OpenStack API calls + if run_api_tests: + test_suite_builder.add_openstack_api_tests(suite, source_filename, ext_net_name, use_keystone=use_keystone, + http_proxy_str=proxy_settings, log_level=log_level) + + # Long running integration type tests + if run_integration_tests: + test_suite_builder.add_openstack_integration_tests(suite, source_filename, ext_net_name, use_keystone=use_keystone, + proxy_settings=proxy_settings, ssh_proxy_cmd=ssh_proxy_cmd, + flavor_metadata=flavor_metadata, + use_floating_ips=use_floating_ips, log_level=log_level) + return suite + + +def main(arguments): + """ + Begins running unit tests. + argv[1] if used must be the source filename else os_env.yaml will be leveraged instead + argv[2] if used must be the proxy server : + """ + logger.info('Starting test suite') + + log_level = LOG_LEVELS.get(arguments.log_level, logging.DEBUG) + + flavor_metadata = None + + if arguments.flavor_metadata: + flavor_metadata = json.loads(arguments.flavor_metadata) + + suite = None + if arguments.env and arguments.ext_net: + suite = __create_test_suite(arguments.env, arguments.ext_net, arguments.proxy, arguments.ssh_proxy_cmd, + arguments.include_unit != ARG_NOT_SET, + arguments.include_connection != ARG_NOT_SET, + arguments.include_api != ARG_NOT_SET, + arguments.include_integration != ARG_NOT_SET, + flavor_metadata, + arguments.use_keystone != ARG_NOT_SET, + arguments.floating_ips != ARG_NOT_SET, log_level) + else: + logger.error('Environment file or external network not defined') + exit(1) + + # To ensure any files referenced via a relative path will begin from the diectory in which this file resides + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + i = 0 + while i < int(arguments.num_runs): + result = unittest.TextTestRunner(verbosity=2).run(suite) + i += 1 + + if result.errors: + logger.error('Number of errors in test suite - ' + str(len(result.errors))) + for test, message in result.errors: + logger.error(str(test) + " ERROR with " + message) + + if result.failures: + logger.error('Number of failures in test suite - ' + str(len(result.failures))) + for test, message in result.failures: + logger.error(str(test) + " FAILED with " + message) + + if (result.errors and len(result.errors) > 0) or (result.failures and len(result.failures) > 0): + logger.error('See above for test failures') + exit(1) + else: + logger.info('All tests completed successfully in run #' + str(i)) + + logger.info('Successful completion of ' + str(i) + ' test runs') + exit(0) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-e', '--env', dest='env', required=True, help='OpenStack credentials file') + parser.add_argument('-n', '--net', dest='ext_net', required=True, help='External network name') + parser.add_argument('-p', '--proxy', dest='proxy', nargs='?', default=None, + help='Optonal HTTP proxy socket (:)') + parser.add_argument('-s', '--ssh-proxy-cmd', dest='ssh_proxy_cmd', nargs='?', default=None, + help='Optonal SSH proxy command value') + parser.add_argument('-l', '--log-level', dest='log_level', default='INFO', + help='Logging Level (FATAL|CRITICAL|ERROR|WARN|INFO|DEBUG)') + + parser.add_argument('-u', '--unit-tests', dest='include_unit', default=ARG_NOT_SET, nargs='?', + help='When argument is set, all tests not requiring OpenStack will be executed') + parser.add_argument('-c', '--connection-tests', dest='include_connection', default=ARG_NOT_SET, nargs='?', + help='When argument is set, simple OpenStack connection tests will be executed') + parser.add_argument('-a', '--api-tests', dest='include_api', default=ARG_NOT_SET, nargs='?', + help='When argument is set, OpenStack API tests will be executed') + parser.add_argument('-i', '--integration-tests', dest='include_integration', default=ARG_NOT_SET, nargs='?', + help='When argument is set, OpenStack integrations tests will be executed') + parser.add_argument('-f', '--floating-ips', dest='floating_ips', default=ARG_NOT_SET, nargs='?', + help='When argument is set, all integration tests requiring Floating IPs will be executed') + + parser.add_argument('-k', '--use-keystone', dest='use_keystone', default=ARG_NOT_SET, nargs='?', + help='When argument is set, the tests will exercise the keystone APIs and must be run on a ' + + 'machine that has access to the admin network' + + ' and is able to create users and groups') + + parser.add_argument('-fm', '--flavor-meta', dest='flavor_metadata', + default='{\"hw:mem_page_size\": \"any\"}', + help='JSON string to be used as flavor metadata for all test instances created') + + parser.add_argument('-r', '--num-runs', dest='num_runs', default=1, + help='Number of test runs to execute (default 1)') + + args = parser.parse_args() + + main(args) diff --git a/snaps/unit_test_suite.py b/snaps/unit_test_suite.py deleted file mode 100644 index ef66a8e..0000000 --- a/snaps/unit_test_suite.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") -# and others. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging -import os -import unittest - -from snaps import test_suite_builder - -__author__ = 'spisarski' - -logger = logging.getLogger('unit_test_suite') - -ARG_NOT_SET = "argument not set" -LOG_LEVELS = {'FATAL': logging.FATAL, 'CRITICAL': logging.CRITICAL, 'ERROR': logging.ERROR, 'WARN': logging.WARN, - 'INFO': logging.INFO, 'DEBUG': logging.DEBUG} - - -def __create_test_suite(source_filename, ext_net_name, proxy_settings, ssh_proxy_cmd, run_unit_tests, flavor_metadata, - use_keystone, use_floating_ips, log_level): - """ - Compiles the tests that should run - :param source_filename: the OpenStack credentials file (required) - :param ext_net_name: the name of the external network to use for floating IPs (required) - :param run_unit_tests: when true, the tests not requiring OpenStack will be added to the test suite - :param proxy_settings: : of the proxy server (optional) - :param ssh_proxy_cmd: the command used to connect via SSH over some proxy server (optional) - :param flavor_metadata: dict() object containing the metadata for flavors created for test VM instance - :param use_keystone: when true, tests creating users and projects will be exercised and must be run on a host that - has access to the cloud's administrative network - :param use_floating_ips: when true, tests requiring floating IPs will be executed - :param log_level: the logging level - :return: - """ - suite = unittest.TestSuite() - - # Tests that do not require a remote connection to an OpenStack cloud - if run_unit_tests: - test_suite_builder.add_unit_tests(suite) - - # Basic connection tests - test_suite_builder.add_openstack_client_tests(suite, source_filename, ext_net_name, use_keystone=use_keystone, - http_proxy_str=proxy_settings, log_level=log_level) - - # Tests the OpenStack API calls - test_suite_builder.add_openstack_api_tests(suite, source_filename, ext_net_name, use_keystone=use_keystone, - http_proxy_str=proxy_settings, log_level=log_level) - - # Long running integration type tests - test_suite_builder.add_openstack_integration_tests(suite, source_filename, ext_net_name, use_keystone=use_keystone, - proxy_settings=proxy_settings, ssh_proxy_cmd=ssh_proxy_cmd, - flavor_metadata=flavor_metadata, - use_floating_ips=use_floating_ips, log_level=log_level) - return suite - - -def main(arguments): - """ - Begins running unit tests. - argv[1] if used must be the source filename else os_env.yaml will be leveraged instead - argv[2] if used must be the proxy server : - """ - logger.info('Starting test suite') - - log_level = LOG_LEVELS.get(arguments.log_level, logging.DEBUG) - - flavor_metadata = None - - if arguments.flavor_metadata: - flavor_metadata = json.loads(arguments.flavor_metadata) - - suite = None - if arguments.env and arguments.ext_net: - suite = __create_test_suite(arguments.env, arguments.ext_net, arguments.proxy, arguments.ssh_proxy_cmd, - arguments.include_units != ARG_NOT_SET, - flavor_metadata, - arguments.use_keystone != ARG_NOT_SET, - arguments.no_floating_ips == ARG_NOT_SET, log_level) - else: - logger.error('Environment file or external network not defined') - exit(1) - - # To ensure any files referenced via a relative path will begin from the diectory in which this file resides - os.chdir(os.path.dirname(os.path.realpath(__file__))) - - result = unittest.TextTestRunner(verbosity=2).run(suite) - - if result.errors: - logger.error('Number of errors in test suite - ' + str(len(result.errors))) - for test, message in result.errors: - logger.error(str(test) + " ERROR with " + message) - - if result.failures: - logger.error('Number of failures in test suite - ' + str(len(result.failures))) - for test, message in result.failures: - logger.error(str(test) + " FAILED with " + message) - - if (result.errors and len(result.errors) > 0) or (result.failures and len(result.failures) > 0): - logger.error('See above for test failures') - exit(1) - else: - logger.info('All tests completed successfully') - - exit(0) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-e', '--env', dest='env', required=True, help='OpenStack credentials file') - parser.add_argument('-n', '--net', dest='ext_net', required=True, help='External network name') - parser.add_argument('-p', '--proxy', dest='proxy', nargs='?', default=None, - help='Optonal HTTP proxy socket (:)') - parser.add_argument('-s', '--ssh-proxy-cmd', dest='ssh_proxy_cmd', nargs='?', default=None, - help='Optonal SSH proxy command value') - parser.add_argument('-l', '--log-level', dest='log_level', default='INFO', - help='Logging Level (FATAL|CRITICAL|ERROR|WARN|INFO|DEBUG)') - parser.add_argument('-k', '--use-keystone', dest='use_keystone', default=ARG_NOT_SET, nargs='?', - help='When argument is set, the tests will exercise the keystone APIs and must be run on a ' + - 'machine that has access to the admin network' + - ' and is able to create users and groups') - parser.add_argument('-f', '--no-floating-ips', dest='no_floating_ips', default=ARG_NOT_SET, nargs='?', - help='When argument is set, all tests requiring Floating IPs will not be executed') - parser.add_argument('-u', '--include-units', dest='include_units', default=ARG_NOT_SET, nargs='?', - help='When argument is set, all tests not requiring OpenStack will be executed') - parser.add_argument('-fm', '--flavor-meta', dest='flavor_metadata', required=False, - default='{\"hw:mem_page_size\": \"any\"}', - help='JSON string to be used as flavor metadata for all test instances created') - args = parser.parse_args() - - main(args) -- cgit 1.2.3-korg