diff options
23 files changed, 981 insertions, 1110 deletions
diff --git a/docs/how-to-use/IntegrationTests.rst b/docs/how-to-use/IntegrationTests.rst index df8859f..b7aa864 100644 --- a/docs/how-to-use/IntegrationTests.rst +++ b/docs/how-to-use/IntegrationTests.rst @@ -606,16 +606,6 @@ create_instance_tests.py - CreateInstanceFromThreePartImage | | Neutron 2 | delete it when using a 3-part image | +-----------------------------------------------------+---------------+-----------------------------------------------------------+ -create_instance_tests.py - CreateInstancePubPrivNetTests (Staging) ------------------------------------------------------------------- - -+---------------------------------------+---------------+-----------------------------------------------------------+ -| Test Name | API Versions | Description | -+=======================================+===============+===========================================================+ -| test_dual_ports_dhcp | Nova 2 | Ensures that a VM with two ports/NICs can have its second | -| | Neutron 2 | NIC configured via SSH/Ansible after startup | -+---------------------------------------+---------------+-----------------------------------------------------------+ - create_instance_tests.py - CreateInstanceIPv6NetworkTests (Staging) ------------------------------------------------------------------- diff --git a/examples/launch.py b/examples/launch.py index 975d834..04bc5d6 100644 --- a/examples/launch.py +++ b/examples/launch.py @@ -18,558 +18,19 @@ # This script is responsible for deploying virtual environments import argparse import logging -import re -import time from jinja2 import Environment, FileSystemLoader import os import yaml from snaps import file_utils -from snaps.config.flavor import FlavorConfig -from snaps.config.image import ImageConfig -from snaps.config.keypair import KeypairConfig -from snaps.config.network import PortConfig, NetworkConfig -from snaps.config.project import ProjectConfig -from snaps.config.qos import QoSConfig -from snaps.config.router import RouterConfig -from snaps.config.security_group import SecurityGroupConfig -from snaps.config.user import UserConfig -from snaps.config.vm_inst import VmInstanceConfig -from snaps.config.volume import VolumeConfig -from snaps.config.volume_type import VolumeTypeConfig -from snaps.openstack.create_flavor import OpenStackFlavor -from snaps.openstack.create_image import OpenStackImage -from snaps.openstack.create_keypairs import OpenStackKeypair -from snaps.openstack.create_network import OpenStackNetwork -from snaps.openstack.create_project import OpenStackProject -from snaps.openstack.create_qos import OpenStackQoS -from snaps.openstack.create_router import OpenStackRouter -from snaps.openstack.create_security_group import OpenStackSecurityGroup -from snaps.openstack.create_user import OpenStackUser -from snaps.openstack.create_volume import OpenStackVolume -from snaps.openstack.create_volume_type import OpenStackVolumeType -from snaps.openstack.os_credentials import OSCreds, ProxySettings -from snaps.openstack.utils import deploy_utils -from snaps.provisioning import ansible_utils +from snaps.openstack.utils import launch_utils __author__ = 'spisarski' logger = logging.getLogger('snaps_launcher') ARG_NOT_SET = "argument not set" -DEFAULT_CREDS_KEY = 'admin' - - -def __get_creds_dict(os_conn_config): - """ - Returns a dict of OSCreds where the key is the creds name. - For backwards compatibility, credentials not contained in a list (only - one) will be returned with the key of None - :param os_conn_config: the credential configuration - :return: a dict of OSCreds objects - """ - if 'connection' in os_conn_config: - return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)} - elif 'connections' in os_conn_config: - out = dict() - for os_conn_dict in os_conn_config['connections']: - config = os_conn_dict.get('connection') - if not config: - raise Exception('Invalid connection format') - - name = config.get('name') - if not name: - raise Exception('Connection config requires a name field') - - out[name] = __get_os_credentials(os_conn_dict) - return out - - -def __get_creds(os_creds_dict, os_user_dict, inst_config): - """ - Returns the appropriate credentials - :param os_creds_dict: a dictionary of OSCreds objects where the name is the - key - :param os_user_dict: a dictionary of OpenStackUser objects where the name - is the key - :param inst_config: - :return: an OSCreds instance or None - """ - os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY) - if 'os_user' in inst_config: - os_user_conf = inst_config['os_user'] - if 'name' in os_user_conf: - user_creator = os_user_dict.get(os_user_conf['name']) - if user_creator: - return user_creator.get_os_creds( - project_name=os_user_conf.get('project_name')) - elif 'os_creds_name' in inst_config: - if 'os_creds_name' in inst_config: - os_creds = os_creds_dict[inst_config['os_creds_name']] - return os_creds - - -def __get_os_credentials(os_conn_config): - """ - Returns an object containing all of the information required to access - OpenStack APIs - :param os_conn_config: The configuration holding the credentials - :return: an OSCreds instance - """ - config = os_conn_config.get('connection') - if not config: - raise Exception('Invalid connection configuration') - - proxy_settings = None - http_proxy = config.get('http_proxy') - if http_proxy: - tokens = re.split(':', http_proxy) - ssh_proxy_cmd = config.get('ssh_proxy_cmd') - proxy_settings = ProxySettings(host=tokens[0], port=tokens[1], - ssh_proxy_cmd=ssh_proxy_cmd) - else: - if 'proxy_settings' in config: - host = config['proxy_settings'].get('host') - port = config['proxy_settings'].get('port') - if host and host != 'None' and port and port != 'None': - proxy_settings = ProxySettings(**config['proxy_settings']) - - if proxy_settings: - config['proxy_settings'] = proxy_settings - else: - if config.get('proxy_settings'): - del config['proxy_settings'] - - return OSCreds(**config) - - -def __parse_ports_config(config): - """ - Parses the "ports" configuration - :param config: The dictionary to parse - :return: a list of PortConfig objects - """ - out = list() - for port_config in config: - out.append(PortConfig(**port_config.get('port'))) - return out - - -def __create_instances(os_creds_dict, creator_class, config_class, config, - config_key, cleanup=False, os_users_dict=None): - """ - Returns a dictionary of SNAPS creator objects where the key is the name - :param os_creds_dict: Dictionary of OSCreds objects where the key is the - name - :param config: The list of configurations for the same type - :param config_key: The list of configurations for the same type - :param cleanup: Denotes whether or not this is being called for cleanup - :return: dictionary - """ - out = {} - - if config: - try: - for config_dict in config: - inst_config = config_dict.get(config_key) - if inst_config: - creator = creator_class( - __get_creds(os_creds_dict, os_users_dict, inst_config), - config_class(**inst_config)) - - if cleanup: - creator.initialize() - else: - creator.create() - out[inst_config['name']] = creator - logger.info('Created configured %s', config_key) - except Exception as e: - logger.error('Unexpected error instantiating creator [%s] ' - 'with exception %s', creator_class, e) - - return out - - -def __create_vm_instances(os_creds_dict, os_users_dict, instances_config, - image_dict, keypairs_dict, cleanup=False): - """ - Returns a dictionary of OpenStackVmInstance objects where the key is the - instance name - :param os_creds_dict: Dictionary of OSCreds objects where the key is the - name - :param os_users_dict: Dictionary of OpenStackUser objects where the key is - the username - :param instances_config: The list of VM instance configurations - :param image_dict: A dictionary of images that will probably be used to - instantiate the VM instance - :param keypairs_dict: A dictionary of keypairs that will probably be used - to instantiate the VM instance - :param cleanup: Denotes whether or not this is being called for cleanup - :return: dictionary - """ - vm_dict = {} - - if instances_config: - try: - for instance_config in instances_config: - conf = instance_config.get('instance') - if conf: - if image_dict: - image_creator = image_dict.get(conf.get('imageName')) - if image_creator: - instance_settings = VmInstanceConfig( - **instance_config['instance']) - kp_creator = keypairs_dict.get( - conf.get('keypair_name')) - vm_dict[conf[ - 'name']] = deploy_utils.create_vm_instance( - __get_creds( - os_creds_dict, os_users_dict, conf), - instance_settings, - image_creator.image_settings, - keypair_creator=kp_creator, - init_only=cleanup) - else: - raise Exception('Image creator instance not found.' - ' Cannot instantiate') - else: - raise Exception('Image dictionary is None. Cannot ' - 'instantiate') - else: - raise Exception('Instance configuration is None. Cannot ' - 'instantiate') - logger.info('Created configured instances') - except Exception as e: - logger.error('Unexpected error creating VM instances - %s', e) - return vm_dict - - -def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict, - image_dict, flavor_dict, env_file): - """ - Applies ansible playbooks to running VMs with floating IPs - :param ansible_configs: a list of Ansible configurations - :param os_creds_dict: Dictionary of OSCreds objects where the key is the - name - :param vm_dict: the dictionary of newly instantiated VMs where the name is - the key - :param image_dict: the dictionary of newly instantiated images where the - name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the - name is the key - :param env_file: the path of the environment for setting the CWD so - playbook location is relative to the deployment file - :return: t/f - true if successful - """ - logger.info("Applying Ansible Playbooks") - if ansible_configs: - # Ensure all hosts are accepting SSH session requests - for vm_inst in list(vm_dict.values()): - if not vm_inst.vm_ssh_active(block=True): - logger.warning( - "Timeout waiting for instance to respond to SSH requests") - return False - - # Set CWD so the deployment file's playbook location can leverage - # relative paths - orig_cwd = os.getcwd() - env_dir = os.path.dirname(env_file) - os.chdir(env_dir) - - # Apply playbooks - for ansible_config in ansible_configs: - if 'pre_sleep_time' in ansible_config: - try: - sleep_time = int(ansible_config['pre_sleep_time']) - logger.info('Waiting %s seconds to apply playbooks', - sleep_time) - time.sleep(sleep_time) - except: - pass - - os_creds = os_creds_dict.get(None, 'admin') - __apply_ansible_playbook(ansible_config, os_creds, vm_dict, - image_dict, flavor_dict) - - # Return to original directory - os.chdir(orig_cwd) - - return True - - -def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, - flavor_dict): - """ - Applies an Ansible configuration setting - :param ansible_config: the configuration settings - :param os_creds: the OpenStack credentials object - :param vm_dict: the dictionary of newly instantiated VMs where the name is - the key - :param image_dict: the dictionary of newly instantiated images where the - name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the - name is the key - """ - if ansible_config: - (remote_user, floating_ips, private_key_filepath, - proxy_settings) = __get_connection_info( - ansible_config, vm_dict) - if floating_ips: - retval = ansible_utils.apply_playbook( - ansible_config['playbook_location'], floating_ips, remote_user, - private_key_filepath, - variables=__get_variables(ansible_config.get('variables'), - os_creds, vm_dict, image_dict, - flavor_dict), - proxy_setting=proxy_settings) - if retval != 0: - # Not a fatal type of event - logger.warning( - 'Unable to apply playbook found at location - %s', - ansible_config.get('playbook_location')) - - -def __get_connection_info(ansible_config, vm_dict): - """ - Returns a tuple of data required for connecting to the running VMs - (remote_user, [floating_ips], private_key_filepath, proxy_settings) - :param ansible_config: the configuration settings - :param vm_dict: the dictionary of VMs where the VM name is the key - :return: tuple where the first element is the user and the second is a list - of floating IPs and the third is the - private key file location and the fourth is an instance of the - snaps.ProxySettings class - (note: in order to work, each of the hosts need to have the same sudo_user - and private key file location values) - """ - if ansible_config.get('hosts'): - hosts = ansible_config['hosts'] - if len(hosts) > 0: - floating_ips = list() - remote_user = None - pk_file = None - proxy_settings = None - for host in hosts: - vm = vm_dict.get(host) - if vm: - fip = vm.get_floating_ip() - if fip: - remote_user = vm.get_image_user() - - if fip: - floating_ips.append(fip.ip) - else: - raise Exception( - 'Could not find floating IP for VM - ' + - vm.name) - - pk_file = vm.keypair_settings.private_filepath - proxy_settings = vm.get_os_creds().proxy_settings - else: - logger.error('Could not locate VM with name - ' + host) - - return remote_user, floating_ips, pk_file, proxy_settings - return None - - -def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict): - """ - Returns a dictionary of substitution variables to be used for Ansible - templates - :param var_config: the variable configuration settings - :param os_creds: the OpenStack credentials object - :param vm_dict: the dictionary of newly instantiated VMs where the name is - the key - :param image_dict: the dictionary of newly instantiated images where the - name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the - name is the key - :return: dictionary or None - """ - if var_config and vm_dict and len(vm_dict) > 0: - variables = dict() - for key, value in var_config.items(): - value = __get_variable_value(value, os_creds, vm_dict, image_dict, - flavor_dict) - if key and value: - variables[key] = value - logger.info( - "Set Jinga2 variable with key [%s] the value [%s]", - key, value) - else: - logger.warning('Key [%s] or Value [%s] must not be None', - str(key), str(value)) - return variables - return None - - -def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict, - flavor_dict): - """ - Returns the associated variable value for use by Ansible for substitution - purposes - :param var_config_values: the configuration dictionary - :param os_creds: the OpenStack credentials object - :param vm_dict: the dictionary of newly instantiated VMs where the name is - the key - :param image_dict: the dictionary of newly instantiated images where the - name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the - name is the key - :return: - """ - if var_config_values['type'] == 'string': - return __get_string_variable_value(var_config_values) - if var_config_values['type'] == 'vm-attr': - return __get_vm_attr_variable_value(var_config_values, vm_dict) - if var_config_values['type'] == 'os_creds': - return __get_os_creds_variable_value(var_config_values, os_creds) - if var_config_values['type'] == 'port': - return __get_vm_port_variable_value(var_config_values, vm_dict) - if var_config_values['type'] == 'floating_ip': - return __get_vm_fip_variable_value(var_config_values, vm_dict) - if var_config_values['type'] == 'image': - return __get_image_variable_value(var_config_values, image_dict) - if var_config_values['type'] == 'flavor': - return __get_flavor_variable_value(var_config_values, flavor_dict) - return None - - -def __get_string_variable_value(var_config_values): - """ - Returns the associated string value - :param var_config_values: the configuration dictionary - :return: the value contained in the dictionary with the key 'value' - """ - return var_config_values['value'] - - -def __get_vm_attr_variable_value(var_config_values, vm_dict): - """ - Returns the associated value contained on a VM instance - :param var_config_values: the configuration dictionary - :param vm_dict: the dictionary containing all VMs where the key is the VM's - name - :return: the value - """ - vm = vm_dict.get(var_config_values['vm_name']) - if vm: - if var_config_values['value'] == 'floating_ip': - return vm.get_floating_ip().ip - if var_config_values['value'] == 'image_user': - return vm.get_image_user() - - -def __get_os_creds_variable_value(var_config_values, os_creds): - """ - Returns the associated OS credentials value - :param var_config_values: the configuration dictionary - :param os_creds: the credentials - :return: the value - """ - logger.info("Retrieving OS Credentials") - if os_creds: - if var_config_values['value'] == 'username': - logger.info("Returning OS username") - return os_creds.username - elif var_config_values['value'] == 'password': - logger.info("Returning OS password") - return os_creds.password - elif var_config_values['value'] == 'auth_url': - logger.info("Returning OS auth_url") - return os_creds.auth_url - elif var_config_values['value'] == 'project_name': - logger.info("Returning OS project_name") - return os_creds.project_name - - logger.info("Returning none") - return None - - -def __get_vm_port_variable_value(var_config_values, vm_dict): - """ - Returns the associated OS credentials value - :param var_config_values: the configuration dictionary - :param vm_dict: the dictionary containing all VMs where the key is the VM's - name - :return: the value - """ - port_name = var_config_values.get('port_name') - vm_name = var_config_values.get('vm_name') - - if port_name and vm_name: - vm = vm_dict.get(vm_name) - if vm: - port_value_id = var_config_values.get('port_value') - if port_value_id: - if port_value_id == 'mac_address': - return vm.get_port_mac(port_name) - if port_value_id == 'ip_address': - return vm.get_port_ip(port_name) - - -def __get_vm_fip_variable_value(var_config_values, vm_dict): - """ - Returns the floating IP value if found - :param var_config_values: the configuration dictionary - :param vm_dict: the dictionary containing all VMs where the key is the VM's - name - :return: the floating IP string value or None - """ - fip_name = var_config_values.get('fip_name') - vm_name = var_config_values.get('vm_name') - - if vm_name: - vm = vm_dict.get(vm_name) - if vm: - fip = vm.get_floating_ip(fip_name) - if fip: - return fip.ip - - -def __get_image_variable_value(var_config_values, image_dict): - """ - Returns the associated image value - :param var_config_values: the configuration dictionary - :param image_dict: the dictionary containing all images where the key is - the name - :return: the value - """ - logger.info("Retrieving image values") - - if image_dict: - if var_config_values.get('image_name'): - image_creator = image_dict.get(var_config_values['image_name']) - if image_creator: - if var_config_values.get('value') and \ - var_config_values['value'] == 'id': - return image_creator.get_image().id - if var_config_values.get('value') and \ - var_config_values['value'] == 'user': - return image_creator.image_settings.image_user - - logger.info("Returning none") - return None - - -def __get_flavor_variable_value(var_config_values, flavor_dict): - """ - Returns the associated flavor value - :param var_config_values: the configuration dictionary - :param flavor_dict: the dictionary containing all flavor creators where the - key is the name - :return: the value or None - """ - logger.info("Retrieving flavor values") - - if flavor_dict: - if var_config_values.get('flavor_name'): - flavor_creator = flavor_dict.get(var_config_values['flavor_name']) - if flavor_creator: - if var_config_values.get('value') and \ - var_config_values['value'] == 'id': - return flavor_creator.get_flavor().id def main(arguments): @@ -605,127 +66,11 @@ def main(arguments): config = yaml.load(output) if config: - os_config = config.get('openstack') - - creators = list() - vm_dict = dict() - images_dict = dict() - flavors_dict = dict() - os_creds_dict = dict() clean = arguments.clean is not ARG_NOT_SET - - if os_config: - os_creds_dict = __get_creds_dict(os_config) - - try: - # Create projects - projects_dict = __create_instances( - os_creds_dict, OpenStackProject, ProjectConfig, - os_config.get('projects'), 'project', clean) - creators.append(projects_dict) - - # Create users - users_dict = __create_instances( - os_creds_dict, OpenStackUser, UserConfig, - os_config.get('users'), 'user', clean) - creators.append(users_dict) - - # Associate new users to projects - if not clean: - for project_creator in projects_dict.values(): - users = project_creator.project_settings.users - for user_name in users: - user_creator = users_dict.get(user_name) - if user_creator: - project_creator.assoc_user( - user_creator.get_user()) - - # Create flavors - flavors_dict = __create_instances( - os_creds_dict, OpenStackFlavor, FlavorConfig, - os_config.get('flavors'), 'flavor', clean, users_dict) - creators.append(flavors_dict) - - # Create QoS specs - qos_dict = __create_instances( - os_creds_dict, OpenStackQoS, QoSConfig, - os_config.get('qos_specs'), 'qos_spec', clean, users_dict) - creators.append(qos_dict) - - # Create volume types - vol_type_dict = __create_instances( - os_creds_dict, OpenStackVolumeType, VolumeTypeConfig, - os_config.get('volume_types'), 'volume_type', clean, - users_dict) - creators.append(vol_type_dict) - - # Create volume types - vol_dict = __create_instances( - os_creds_dict, OpenStackVolume, VolumeConfig, - os_config.get('volumes'), 'volume', clean, users_dict) - creators.append(vol_dict) - - # Create images - images_dict = __create_instances( - os_creds_dict, OpenStackImage, ImageConfig, - os_config.get('images'), 'image', clean, users_dict) - creators.append(images_dict) - - # Create networks - creators.append(__create_instances( - os_creds_dict, OpenStackNetwork, NetworkConfig, - os_config.get('networks'), 'network', clean, users_dict)) - - # Create routers - creators.append(__create_instances( - os_creds_dict, OpenStackRouter, RouterConfig, - os_config.get('routers'), 'router', clean, users_dict)) - - # Create keypairs - keypairs_dict = __create_instances( - os_creds_dict, OpenStackKeypair, KeypairConfig, - os_config.get('keypairs'), 'keypair', clean, users_dict) - creators.append(keypairs_dict) - - # Create security groups - creators.append(__create_instances( - os_creds_dict, OpenStackSecurityGroup, - SecurityGroupConfig, - os_config.get('security_groups'), 'security_group', clean, - users_dict)) - - # Create instance - vm_dict = __create_vm_instances( - os_creds_dict, users_dict, os_config.get('instances'), - images_dict, keypairs_dict, - arguments.clean is not ARG_NOT_SET) - creators.append(vm_dict) - logger.info( - 'Completed creating/retrieving all configured instances') - except Exception as e: - logger.error( - 'Unexpected error deploying environment. Rolling back due' - ' to - ' + str(e)) - raise - - # Must enter either block - if arguments.clean is not ARG_NOT_SET: - # Cleanup Environment - __cleanup(creators, arguments.clean_image is not ARG_NOT_SET) - elif arguments.deploy is not ARG_NOT_SET: - logger.info('Configuring NICs where required') - for vm in vm_dict.values(): - vm.config_nics() - logger.info('Completed NIC configuration') - - # Provision VMs - ansible_config = config.get('ansible') - if ansible_config and vm_dict: - if not __apply_ansible_playbooks(ansible_config, - os_creds_dict, vm_dict, - images_dict, flavors_dict, - arguments.tmplt_file): - logger.error("Problem applying ansible playbooks") + clean_image = arguments.clean_image is not ARG_NOT_SET + deploy = arguments.deploy is not ARG_NOT_SET + launch_utils.launch_config( + config, arguments.tmplt_file, deploy, clean, clean_image) else: logger.error( 'Unable to read configuration file - ' + arguments.tmplt_file) @@ -734,17 +79,6 @@ def main(arguments): exit(0) -def __cleanup(creators, clean_image=False): - for creator_dict in reversed(creators): - for key, creator in creator_dict.items(): - if ((isinstance(creator, OpenStackImage) and clean_image) - or not isinstance(creator, OpenStackImage)): - try: - creator.clean() - except Exception as e: - logger.warning('Error cleaning component - %s', e) - - if __name__ == '__main__': # To ensure any files referenced via a relative path will begin from the # directory in which this file resides diff --git a/snaps/config/tests/vm_inst_tests.py b/snaps/config/tests/vm_inst_tests.py index 71d2e0b..d7fb287 100644 --- a/snaps/config/tests/vm_inst_tests.py +++ b/snaps/config/tests/vm_inst_tests.py @@ -64,6 +64,7 @@ class VmInstanceConfigUnitTests(unittest.TestCase): self.assertEqual(900, settings.vm_boot_timeout) self.assertEqual(300, settings.vm_delete_timeout) self.assertEqual(180, settings.ssh_connect_timeout) + self.assertEqual(300, settings.cloud_init_timeout) self.assertIsNone(settings.availability_zone) self.assertIsNone(settings.volume_names) @@ -82,6 +83,7 @@ class VmInstanceConfigUnitTests(unittest.TestCase): self.assertEqual(900, settings.vm_boot_timeout) self.assertEqual(300, settings.vm_delete_timeout) self.assertEqual(180, settings.ssh_connect_timeout) + self.assertEqual(300, settings.cloud_init_timeout) self.assertIsNone(settings.availability_zone) self.assertIsNone(settings.volume_names) @@ -95,8 +97,8 @@ class VmInstanceConfigUnitTests(unittest.TestCase): security_group_names=['sec_grp_1'], floating_ip_settings=[fip_settings], sudo_user='joe', vm_boot_timeout=999, vm_delete_timeout=333, - ssh_connect_timeout=111, availability_zone='server name', - volume_names=['vol1']) + ssh_connect_timeout=111, cloud_init_timeout=998, + availability_zone='server name', volume_names=['vol1']) self.assertEqual('foo', settings.name) self.assertEqual('bar', settings.flavor) self.assertEqual(1, len(settings.port_settings)) @@ -114,6 +116,7 @@ class VmInstanceConfigUnitTests(unittest.TestCase): self.assertEqual(999, settings.vm_boot_timeout) self.assertEqual(333, settings.vm_delete_timeout) self.assertEqual(111, settings.ssh_connect_timeout) + self.assertEqual(998, settings.cloud_init_timeout) self.assertEqual('server name', settings.availability_zone) self.assertEqual('vol1', settings.volume_names[0]) @@ -127,8 +130,8 @@ class VmInstanceConfigUnitTests(unittest.TestCase): 'security_group_names': ['sec_grp_1'], 'floating_ips': [fip_settings], 'sudo_user': 'joe', 'vm_boot_timeout': 999, 'vm_delete_timeout': 333, - 'ssh_connect_timeout': 111, 'availability_zone': 'server name', - 'volume_names': ['vol2']}) + 'ssh_connect_timeout': 111, 'cloud_init_timeout': 998, + 'availability_zone': 'server name', 'volume_names': ['vol2']}) self.assertEqual('foo', settings.name) self.assertEqual('bar', settings.flavor) self.assertEqual(1, len(settings.port_settings)) @@ -145,6 +148,7 @@ class VmInstanceConfigUnitTests(unittest.TestCase): self.assertEqual(999, settings.vm_boot_timeout) self.assertEqual(333, settings.vm_delete_timeout) self.assertEqual(111, settings.ssh_connect_timeout) + self.assertEqual(998, settings.cloud_init_timeout) self.assertEqual('server name', settings.availability_zone) self.assertEqual('vol2', settings.volume_names[0]) diff --git a/snaps/config/vm_inst.py b/snaps/config/vm_inst.py index 9533ea1..6a63e33 100644 --- a/snaps/config/vm_inst.py +++ b/snaps/config/vm_inst.py @@ -32,12 +32,14 @@ class VmInstanceConfig(object): :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to connect to the VM - :param vm_boot_timeout: the amount of time a thread will sleep waiting + :param vm_boot_timeout: the amount of time a thread will wait for an instance to boot - :param vm_delete_timeout: the amount of time a thread will sleep - waiting for an instance to be deleted - :param ssh_connect_timeout: the amount of time a thread will sleep - waiting obtaining an SSH connection to a VM + :param vm_delete_timeout: the amount of time a thread will wait + for an instance to be deleted + :param ssh_connect_timeout: the amount of time a thread will wait + to obtain an SSH connection to a VM + :param cloud_init_timeout: the amount of time a thread will wait for + cloud-init to complete :param availability_zone: the name of the compute server on which to deploy the VM (optional) :param volume_names: a list of the names of the volume to attach @@ -93,6 +95,7 @@ class VmInstanceConfig(object): self.vm_boot_timeout = kwargs.get('vm_boot_timeout', 900) self.vm_delete_timeout = kwargs.get('vm_delete_timeout', 300) self.ssh_connect_timeout = kwargs.get('ssh_connect_timeout', 180) + self.cloud_init_timeout = kwargs.get('cloud_init_timeout', 300) self.availability_zone = kwargs.get('availability_zone') self.volume_names = kwargs.get('volume_names') diff --git a/snaps/openstack/create_flavor.py b/snaps/openstack/create_flavor.py index b866d43..65b9059 100644 --- a/snaps/openstack/create_flavor.py +++ b/snaps/openstack/create_flavor.py @@ -71,8 +71,6 @@ class OpenStackFlavor(OpenStackComputeObject): if self.flavor_settings.metadata: nova_utils.set_flavor_keys(self._nova, self.__flavor, self.flavor_settings.metadata) - else: - logger.info('Did not create flavor due to cleanup mode') return self.__flavor diff --git a/snaps/openstack/create_image.py b/snaps/openstack/create_image.py index 2e7aa39..a5520e3 100644 --- a/snaps/openstack/create_image.py +++ b/snaps/openstack/create_image.py @@ -57,6 +57,7 @@ class OpenStackImage(OpenStackCloudObject): self.__glance = glance_utils.glance_client(self._os_creds) self.__image = glance_utils.get_image( self.__glance, image_settings=self.image_settings) + if self.__image: logger.info('Found image with name - ' + self.image_settings.name) return self.__image @@ -70,6 +71,7 @@ class OpenStackImage(OpenStackCloudObject): self.__kernel_image = glance_utils.get_image( self.__glance, image_settings=self.image_settings.kernel_image_settings) + if self.image_settings.ramdisk_image_settings: self.__ramdisk_image = glance_utils.get_image( self.__glance, @@ -97,6 +99,7 @@ class OpenStackImage(OpenStackCloudObject): self.__glance, self.image_settings.kernel_image_settings) extra_properties['kernel_id'] = self.__kernel_image.id + if self.image_settings.ramdisk_image_settings: if not self.__ramdisk_image: logger.info( @@ -113,6 +116,7 @@ class OpenStackImage(OpenStackCloudObject): logger.info( 'Created image with name - %s', self.image_settings.name) + if self.__image and self.image_active(block=True): logger.info( 'Image is now active with name - %s', @@ -122,8 +126,6 @@ class OpenStackImage(OpenStackCloudObject): raise ImageCreationError( 'Image was not created or activated in the alloted amount' 'of time') - else: - logger.info('Did not create image due to cleanup mode') return self.__image diff --git a/snaps/openstack/create_instance.py b/snaps/openstack/create_instance.py index 336c936..d91e360 100644 --- a/snaps/openstack/create_instance.py +++ b/snaps/openstack/create_instance.py @@ -15,7 +15,6 @@ import logging import time -from neutronclient.common.exceptions import PortNotFoundClient from novaclient.exceptions import NotFound, BadRequest from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig @@ -255,19 +254,25 @@ class OpenStackVmInstance(OpenStackComputeObject): # Cleanup floating IPs for name, floating_ip in self.__floating_ip_dict.items(): - try: - logger.info('Deleting Floating IP - ' + floating_ip.ip) - neutron_utils.delete_floating_ip(self.__neutron, floating_ip) - except Exception as e: - logger.error('Error deleting Floating IP - ' + str(e)) + logger.info('Deleting Floating IP - ' + floating_ip.ip) + neutron_utils.delete_floating_ip(self.__neutron, floating_ip) + self.__floating_ip_dict = dict() - # Detach Volume - for volume_rec in self.__vm.volume_ids: - cinder = cinder_utils.cinder_client(self._os_creds) - volume = cinder_utils.get_volume_by_id(cinder, volume_rec['id']) - if volume: - try: + # Cleanup ports + for name, port in self.__ports: + logger.info('Deleting Port with ID - %s ', port.id) + neutron_utils.delete_port(self.__neutron, port) + + self.__ports = list() + + if self.__vm: + # Detach Volume + for volume_rec in self.__vm.volume_ids: + cinder = cinder_utils.cinder_client(self._os_creds) + volume = cinder_utils.get_volume_by_id( + cinder, volume_rec['id']) + if volume: vm = nova_utils.detach_volume( self._nova, self.__neutron, self.__vm, volume, 30) if vm: @@ -275,50 +280,32 @@ class OpenStackVmInstance(OpenStackComputeObject): else: logger.warn( 'Timeout waiting to detach volume %s', volume.name) - except Exception as e: - logger.error('Unexpected error detaching volume %s ' - 'with error %s', volume.name, e) - else: - logger.warn('Unable to detach volume with ID - [%s]', - volume_rec['id']) + else: + logger.warn('Unable to detach volume with ID - [%s]', + volume_rec['id']) - # Cleanup ports - for name, port in self.__ports: - logger.info('Deleting Port with ID - %s ', port.id) - try: - neutron_utils.delete_port(self.__neutron, port) - except PortNotFoundClient as e: - logger.warning('Unexpected error deleting port - %s', e) - pass - self.__ports = list() + # Cleanup VM + logger.info( + 'Deleting VM instance - ' + self.instance_settings.name) - # Cleanup VM - if self.__vm: try: - logger.info( - 'Deleting VM instance - ' + self.instance_settings.name) nova_utils.delete_vm_instance(self._nova, self.__vm) - except Exception as e: - logger.error('Error deleting VM - %s', e) + except NotFound as e: + logger.warn('Instance already deleted - %s', e) # Block until instance cannot be found or returns the status of # DELETED logger.info('Checking deletion status') - try: - if self.vm_deleted(block=True): - logger.info( - 'VM has been properly deleted VM with name - %s', - self.instance_settings.name) - self.__vm = None - else: - logger.error( - 'VM not deleted within the timeout period of %s ' - 'seconds', self.instance_settings.vm_delete_timeout) - except Exception as e: + if self.vm_deleted(block=True): + logger.info( + 'VM has been properly deleted VM with name - %s', + self.instance_settings.name) + self.__vm = None + else: logger.error( - 'Unexpected error while checking VM instance status - %s', - e) + 'VM not deleted within the timeout period of %s ' + 'seconds', self.instance_settings.vm_delete_timeout) def __query_ports(self, port_settings): """ @@ -489,25 +476,6 @@ class OpenStackVmInstance(OpenStackComputeObject): """ return nova_utils.get_server_info(self._nova, self.__vm) - def config_nics(self): - """ - Responsible for configuring NICs on RPM systems where the instance has - more than one configured port - :return: the value returned by ansible_utils.apply_ansible_playbook() - """ - if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0: - if self.vm_active(block=True) and self.vm_ssh_active(block=True): - for key, port in self.__ports: - port_index = self.__ports.index((key, port)) - if port_index > 0: - nic_name = 'eth' + repr(port_index) - retval = self.__config_nic( - nic_name, port, - self.__get_first_provisioning_floating_ip().ip) - logger.info('Configured NIC - %s on VM - %s', - nic_name, self.instance_settings.name) - return retval - def __get_first_provisioning_floating_ip(self): """ Returns the first floating IP tagged with the Floating IP name if @@ -528,31 +496,6 @@ class OpenStackVmInstance(OpenStackComputeObject): for key, fip in self.__floating_ip_dict.items(): return fip - def __config_nic(self, nic_name, port, ip): - """ - Although ports/NICs can contain multiple IPs, this code currently only - supports the first. - - :param nic_name: Name of the interface - :param port: The port information containing the expected IP values. - :param ip: The IP on which to apply the playbook. - :return: the return value from ansible - """ - port_ip = port.ips[0]['ip_address'] - variables = { - 'floating_ip': ip, - 'nic_name': nic_name, - 'nic_ip': port_ip - } - - if self.image_settings.nic_config_pb_loc and self.keypair_settings: - return self.apply_ansible_playbook( - self.image_settings.nic_config_pb_loc, variables) - else: - logger.warning( - 'VM %s cannot self configure NICs eth1++. No playbook or ' - 'keypairs found.', self.instance_settings.name) - def apply_ansible_playbook(self, pb_file_loc, variables=None, fip_name=None): """ @@ -722,6 +665,56 @@ class OpenStackVmInstance(OpenStackComputeObject): return True return False + def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL): + """ + Returns true when the VM's cloud-init routine has completed. + Note: this is currently done via SSH, therefore, if this instance does + not have a Floating IP or a running SSH server, this routine + will always return False or raise an Exception + :param block: When true, thread will block until active or timeout + value in seconds has been exceeded (False) + :param poll_interval: The polling interval + :return: T/F + """ + # sleep and wait for VM status change + logger.info('Checking if cloud-init has completed') + + timeout = self.instance_settings.cloud_init_timeout + + if self.vm_active(block=True) and self.vm_ssh_active(block=True): + if block: + start = time.time() + else: + start = time.time() - timeout + + while timeout > time.time() - start: + status = self.__cloud_init_complete() + if status: + logger.info('cloud-init complete for VM instance') + return True + + logger.debug('Retry cloud-init query in ' + str( + poll_interval) + ' seconds') + time.sleep(poll_interval) + logger.debug('cloud-init complete timeout in ' + str( + timeout - (time.time() - start))) + + logger.error('Timeout waiting for cloud-init to complete') + return False + + def __cloud_init_complete(self): + """ + Returns True when can create a SSH session else False + :return: T/F + """ + if len(self.__floating_ip_dict) > 0: + ssh = self.ssh_client() + if ssh: + stdin1, stdout1, sterr1 = ssh.exec_command( + 'ls -l /var/lib/cloud/instance/boot-finished') + return stdout1.channel.recv_exit_status() == 0 + return False + def get_floating_ip(self, fip_name=None): """ Returns the floating IP object byt name if found, else the first known, diff --git a/snaps/openstack/tests/create_instance_tests.py b/snaps/openstack/tests/create_instance_tests.py index 486018d..55da144 100644 --- a/snaps/openstack/tests/create_instance_tests.py +++ b/snaps/openstack/tests/create_instance_tests.py @@ -306,7 +306,9 @@ class SimpleHealthCheck(OSIntegrationTestCase): self.inst_creator = None self.priv_net_config = openstack_tests.get_priv_net_config( - net_name=guid + '-priv-net', subnet_name=guid + '-priv-subnet') + net_name=guid + '-priv-net', + subnet_name=guid + '-priv-subnet', + netconf_override=self.netconf_override) self.port_settings = PortConfig( name=self.port_1_name, network_name=self.priv_net_config.network_settings.name) @@ -424,7 +426,8 @@ class CreateInstanceSimpleTests(OSIntegrationTestCase): net_config = openstack_tests.get_priv_net_config( net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet', - router_name=guid + '-pub-router', external_net=self.ext_net_name) + router_name=guid + '-pub-router', external_net=self.ext_net_name, + netconf_override=self.netconf_override) # Initialize for tearDown() self.image_creator = None @@ -558,7 +561,8 @@ class CreateInstanceSingleNetworkTests(OSIntegrationTestCase): self.pub_net_config = openstack_tests.get_pub_net_config( net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet', - router_name=guid + '-pub-router', external_net=self.ext_net_name) + router_name=guid + '-pub-router', external_net=self.ext_net_name, + netconf_override=self.netconf_override) os_image_settings = openstack_tests.cirros_image_settings( name=guid + '-image', image_metadata=self.image_metadata) try: @@ -1199,7 +1203,8 @@ class CreateInstancePortManipulationTests(OSIntegrationTestCase): self.net_config = openstack_tests.get_priv_net_config( net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet', - router_name=guid + '-pub-router', external_net=self.ext_net_name) + router_name=guid + '-pub-router', external_net=self.ext_net_name, + netconf_override=self.netconf_override) os_image_settings = openstack_tests.cirros_image_settings( name=guid + '-image', image_metadata=self.image_metadata) @@ -1496,7 +1501,8 @@ class CreateInstanceOnComputeHost(OSIntegrationTestCase): self.inst_creators = list() self.priv_net_config = openstack_tests.get_priv_net_config( - net_name=guid + '-priv-net', subnet_name=guid + '-priv-subnet') + net_name=guid + '-priv-net', subnet_name=guid + '-priv-subnet', + netconf_override=self.netconf_override) os_image_settings = openstack_tests.cirros_image_settings( name=guid + '-image', image_metadata=self.image_metadata) @@ -1601,224 +1607,6 @@ class CreateInstanceOnComputeHost(OSIntegrationTestCase): index += 1 -class CreateInstancePubPrivNetTests(OSIntegrationTestCase): - """ - Test for the CreateInstance class with two NIC/Ports, eth0 with floating IP - and eth1 w/o. - These tests require a Centos image - """ - - def setUp(self): - """ - Instantiates the CreateImage object that is responsible for downloading - and creating an OS image file within OpenStack - """ - super(self.__class__, self).__start__() - - self.nova = nova_utils.nova_client(self.os_creds) - - # Initialize for tearDown() - self.image_creator = None - self.network_creators = list() - self.router_creators = list() - self.flavor_creator = None - self.keypair_creator = None - self.sec_grp_creator = None - self.inst_creator = None - - self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4()) - self.keypair_priv_filepath = 'tmp/' + self.guid - self.keypair_pub_filepath = self.keypair_priv_filepath + '.pub' - self.keypair_name = self.guid + '-kp' - self.vm_inst_name = self.guid + '-inst' - self.port_1_name = self.guid + '-port-1' - self.port_2_name = self.guid + '-port-2' - self.floating_ip_name = self.guid + 'fip1' - self.priv_net_config = openstack_tests.get_priv_net_config( - net_name=self.guid + '-priv-net', - subnet_name=self.guid + '-priv-subnet', - router_name=self.guid + '-priv-router', - external_net=self.ext_net_name) - self.pub_net_config = openstack_tests.get_pub_net_config( - net_name=self.guid + '-pub-net', - subnet_name=self.guid + '-pub-subnet', - router_name=self.guid + '-pub-router', - external_net=self.ext_net_name) - - image_name = self.__class__.__name__ + '-' + str(uuid.uuid4()) - os_image_settings = openstack_tests.centos_image_settings( - name=image_name, image_metadata=self.image_metadata) - - try: - # Create Image - self.image_creator = OpenStackImage(self.os_creds, - os_image_settings) - self.image_creator.create() - - # First network is public - self.network_creators.append(OpenStackNetwork( - self.os_creds, self.pub_net_config.network_settings)) - # Second network is private - self.network_creators.append(OpenStackNetwork( - self.os_creds, self.priv_net_config.network_settings)) - for network_creator in self.network_creators: - network_creator.create() - - self.router_creators.append(OpenStackRouter( - self.os_creds, self.pub_net_config.router_settings)) - self.router_creators.append(OpenStackRouter( - self.os_creds, self.priv_net_config.router_settings)) - - # Create Routers - for router_creator in self.router_creators: - router_creator.create() - - # Create Flavor - self.flavor_creator = OpenStackFlavor( - self.admin_os_creds, - FlavorConfig(name=self.guid + '-flavor-name', ram=512, - disk=10, vcpus=2, - metadata=self.flavor_metadata)) - self.flavor_creator.create() - - # Create Keypair - self.keypair_creator = OpenStackKeypair( - self.os_creds, KeypairConfig( - name=self.keypair_name, - public_filepath=self.keypair_pub_filepath, - private_filepath=self.keypair_priv_filepath)) - self.keypair_creator.create() - - sec_grp_name = self.guid + '-sec-grp' - rule1 = SecurityGroupRuleConfig( - sec_grp_name=sec_grp_name, direction=Direction.ingress, - protocol=Protocol.icmp) - rule2 = SecurityGroupRuleConfig( - sec_grp_name=sec_grp_name, direction=Direction.ingress, - protocol=Protocol.tcp, port_range_min=22, port_range_max=22) - self.sec_grp_creator = OpenStackSecurityGroup( - self.os_creds, - SecurityGroupConfig( - name=sec_grp_name, rule_settings=[rule1, rule2])) - self.sec_grp_creator.create() - except: - self.tearDown() - raise - - def tearDown(self): - """ - Cleans the created objects - """ - if self.inst_creator: - try: - self.inst_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning VM instance with message ' - '- %s', e) - - if self.keypair_creator: - try: - self.keypair_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning keypair with message - %s', - e) - - if self.flavor_creator: - try: - self.flavor_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning flavor with message - %s', - e) - - for router_creator in self.router_creators: - try: - router_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning router with message - %s', - e) - - for network_creator in self.network_creators: - try: - network_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning network with message - %s', - e) - - if self.sec_grp_creator: - try: - self.sec_grp_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning security group with message' - ' - %s', e) - - if self.image_creator and not self.image_creator.image_settings.exists: - try: - self.image_creator.clean() - except Exception as e: - logger.error( - 'Unexpected exception cleaning image with message - %s', e) - - super(self.__class__, self).__clean__() - - def test_dual_ports_dhcp(self): - """ - Tests the creation of an OpenStack instance with a dual ports/NICs with - a DHCP assigned IP. - NOTE: This test and any others that call ansible will most likely fail - unless you do one of two things: - 1. Have a ~/.ansible.cfg (or alternate means) to - set host_key_checking = False - 2. Set the following environment variable in your executing shell: - ANSIBLE_HOST_KEY_CHECKING=False - Should this not be performed, the creation of the host ssh key will - cause your ansible calls to fail. - """ - # Create ports/NICs for instance - ports_settings = [] - ctr = 1 - for network_creator in self.network_creators: - ports_settings.append(PortConfig( - name=self.guid + '-port-' + str(ctr), - network_name=network_creator.network_settings.name)) - ctr += 1 - - # Create instance - instance_settings = VmInstanceConfig( - name=self.vm_inst_name, - flavor=self.flavor_creator.flavor_settings.name, - port_settings=ports_settings, - security_group_names=[self.sec_grp_creator.sec_grp_settings.name], - floating_ip_settings=[FloatingIpConfig( - name=self.floating_ip_name, port_name=self.port_1_name, - router_name=self.pub_net_config.router_settings.name)]) - - self.inst_creator = OpenStackVmInstance( - self.os_creds, instance_settings, - self.image_creator.image_settings, - keypair_settings=self.keypair_creator.keypair_settings) - - vm_inst = self.inst_creator.create(block=True) - - self.assertEqual(vm_inst.id, self.inst_creator.get_vm_inst().id) - - # Effectively blocks until VM has been properly activated - self.assertTrue(self.inst_creator.vm_active(block=True)) - - ip = self.inst_creator.get_port_ip(ports_settings[0].name) - self.assertTrue(check_dhcp_lease(self.inst_creator, ip)) - - # Effectively blocks until VM's ssh port has been opened - self.assertTrue(self.inst_creator.vm_ssh_active(block=True)) - - self.assertEqual(0, self.inst_creator.config_nics()) - - class InstanceSecurityGroupTests(OSIntegrationTestCase): """ Tests that include, add, and remove security groups from VM instances @@ -1846,7 +1634,8 @@ class InstanceSecurityGroupTests(OSIntegrationTestCase): net_name=self.guid + '-pub-net', subnet_name=self.guid + '-pub-subnet', router_name=self.guid + '-pub-router', - external_net=self.ext_net_name) + external_net=self.ext_net_name, + netconf_override=self.netconf_override) # Initialize for tearDown() self.image_creator = None @@ -2167,7 +1956,8 @@ class CreateInstanceFromThreePartImage(OSIntegrationTestCase): net_config = openstack_tests.get_priv_net_config( net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet', - router_name=guid + '-pub-router', external_net=self.ext_net_name) + router_name=guid + '-pub-router', external_net=self.ext_net_name, + netconf_override=self.netconf_override) # Initialize for tearDown() self.image_creator = None @@ -3023,7 +2813,8 @@ class CreateInstanceVolumeTests(OSIntegrationTestCase): net_config = openstack_tests.get_priv_net_config( net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet', - router_name=guid + '-pub-router', external_net=self.ext_net_name) + router_name=guid + '-pub-router', external_net=self.ext_net_name, + netconf_override=self.netconf_override) self.volume_settings1 = VolumeConfig( name=self.__class__.__name__ + '-' + str(guid) + '-1') diff --git a/snaps/openstack/tests/create_network_tests.py b/snaps/openstack/tests/create_network_tests.py index ac0aad1..966cbd0 100644 --- a/snaps/openstack/tests/create_network_tests.py +++ b/snaps/openstack/tests/create_network_tests.py @@ -366,7 +366,8 @@ class CreateNetworkSuccessTests(OSIntegrationTestCase): guid = self.__class__.__name__ + '-' + str(uuid.uuid4()) self.net_config = openstack_tests.get_pub_net_config( net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet', - router_name=guid + '-pub-router', external_net=self.ext_net_name) + router_name=guid + '-pub-router', external_net=self.ext_net_name, + netconf_override=self.netconf_override) self.neutron = neutron_utils.neutron_client(self.os_creds) diff --git a/snaps/openstack/tests/openstack_tests.py b/snaps/openstack/tests/openstack_tests.py index 4b00922..a3dec11 100644 --- a/snaps/openstack/tests/openstack_tests.py +++ b/snaps/openstack/tests/openstack_tests.py @@ -271,15 +271,12 @@ def centos_image_settings(name, url=None, image_metadata=None, else: metadata = image_metadata - pb_path = pkg_resources.resource_filename( - 'snaps.provisioning.ansible_pb.centos-network-setup.playbooks', - 'configure_host.yml') return create_image_settings( image_name=name, image_user=CENTOS_USER, image_format=DEFAULT_IMAGE_FORMAT, metadata=metadata, disk_url=url, default_url=CENTOS_DEFAULT_IMAGE_URL, kernel_settings=kernel_settings, ramdisk_settings=ramdisk_settings, - public=public, nic_config_pb_loc=pb_path) + public=public) def ubuntu_image_settings(name, url=None, image_metadata=None, @@ -304,43 +301,59 @@ def ubuntu_image_settings(name, url=None, image_metadata=None, else: metadata = image_metadata - pb_path = pkg_resources.resource_filename( - 'snaps.provisioning.ansible_pb.ubuntu-network-setup.playbooks', - 'configure_host.yml') return create_image_settings( image_name=name, image_user=UBUNTU_USER, image_format=DEFAULT_IMAGE_FORMAT, metadata=metadata, disk_url=url, default_url=UBUNTU_DEFAULT_IMAGE_URL, kernel_settings=kernel_settings, ramdisk_settings=ramdisk_settings, - public=public, nic_config_pb_loc=pb_path) + public=public) def get_priv_net_config(net_name, subnet_name, router_name=None, - cidr='10.55.0.0/24', external_net=None): + cidr='10.55.0.0/24', external_net=None, + netconf_override=None): return OSNetworkConfig(net_name, subnet_name, cidr, router_name, - external_gateway=external_net) + external_gateway=external_net, + netconf_override=netconf_override) def get_pub_net_config(net_name, subnet_name=None, router_name=None, - cidr='10.55.1.0/24', external_net=None): + cidr='10.55.1.0/24', external_net=None, + netconf_override=None): return OSNetworkConfig(net_name, subnet_name, cidr, router_name, - external_gateway=external_net) + external_gateway=external_net, + netconf_override=netconf_override) class OSNetworkConfig: """ Represents the settings required for the creation of a network in OpenStack + where netconf_override is used to reconfigure the network_type, + physical_network and segmentation_id """ def __init__(self, net_name, subnet_name=None, subnet_cidr=None, - router_name=None, external_gateway=None): - + router_name=None, external_gateway=None, + netconf_override=None): + """ + :param netconf_override: dict() containing the reconfigured network_type, + physical_network and segmentation_id + """ + + network_conf = None if subnet_name and subnet_cidr: - self.network_settings = NetworkConfig( + network_conf = NetworkConfig( name=net_name, subnet_settings=[ SubnetConfig(cidr=subnet_cidr, name=subnet_name)]) else: - self.network_settings = NetworkConfig(name=net_name) + network_conf = NetworkConfig(name=net_name) + if netconf_override: + network_conf.network_type = netconf_override.get('network_type') + network_conf.physical_network = netconf_override.get( + 'physical_network') + network_conf.segmentation_id = netconf_override.get( + 'segmentation_id') + self.network_settings = network_conf if router_name: if subnet_name: diff --git a/snaps/openstack/tests/os_source_file_test.py b/snaps/openstack/tests/os_source_file_test.py index ef4fcfa..7e910a4 100644 --- a/snaps/openstack/tests/os_source_file_test.py +++ b/snaps/openstack/tests/os_source_file_test.py @@ -80,7 +80,7 @@ class OSIntegrationTestCase(OSComponentTestCase): def __init__(self, method_name='runTest', os_creds=None, ext_net_name=None, use_keystone=True, flavor_metadata=None, image_metadata=None, - log_level=logging.DEBUG): + netconf_override=None, log_level=logging.DEBUG): """ Super for integration tests requiring a connection to OpenStack :param method_name: default 'runTest' @@ -98,12 +98,15 @@ class OSIntegrationTestCase(OSComponentTestCase): 'ramdisk_url': '{URI}/cirros-0.3.4-x86_64-initramfs'}) :param flavor_metadata: dict() to be sent directly into the Nova client generally used for page sizes + :param netconf_override: dict() containing the configured network_type, + physical_network and segmentation_id :param log_level: the logging level of your test run (default DEBUG) """ super(OSIntegrationTestCase, self).__init__( method_name=method_name, os_creds=os_creds, ext_net_name=ext_net_name, image_metadata=image_metadata, log_level=log_level) + self.netconf_override = netconf_override self.use_keystone = use_keystone self.keystone = None self.flavor_metadata = flavor_metadata @@ -111,7 +114,8 @@ class OSIntegrationTestCase(OSComponentTestCase): @staticmethod def parameterize(testcase_klass, os_creds, ext_net_name, use_keystone=False, flavor_metadata=None, - image_metadata=None, log_level=logging.DEBUG): + image_metadata=None, netconf_override=None, + log_level=logging.DEBUG): """ Create a suite containing all tests taken from the given subclass, passing them the parameter 'param'. @@ -122,7 +126,8 @@ class OSIntegrationTestCase(OSComponentTestCase): for name in test_names: suite.addTest(testcase_klass(name, os_creds, ext_net_name, use_keystone, flavor_metadata, - image_metadata, log_level)) + image_metadata, netconf_override, + log_level)) return suite """ diff --git a/snaps/openstack/utils/launch_utils.py b/snaps/openstack/utils/launch_utils.py new file mode 100644 index 0000000..abf04b5 --- /dev/null +++ b/snaps/openstack/utils/launch_utils.py @@ -0,0 +1,804 @@ +# +# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") +# and others. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This utility makes it easy to create OpenStack objects +import logging +import re +import socket +import struct + +import os +from keystoneauth1.exceptions import Unauthorized + +from snaps.config.flavor import FlavorConfig +from snaps.config.image import ImageConfig +from snaps.config.keypair import KeypairConfig +from snaps.config.network import PortConfig, NetworkConfig +from snaps.config.project import ProjectConfig +from snaps.config.qos import QoSConfig +from snaps.config.router import RouterConfig +from snaps.config.security_group import SecurityGroupConfig +from snaps.config.user import UserConfig +from snaps.config.vm_inst import VmInstanceConfig +from snaps.config.volume import VolumeConfig +from snaps.config.volume_type import VolumeTypeConfig +from snaps.openstack.create_flavor import OpenStackFlavor +from snaps.openstack.create_image import OpenStackImage +from snaps.openstack.create_keypairs import OpenStackKeypair +from snaps.openstack.create_network import OpenStackNetwork +from snaps.openstack.create_project import OpenStackProject +from snaps.openstack.create_qos import OpenStackQoS +from snaps.openstack.create_router import OpenStackRouter +from snaps.openstack.create_security_group import OpenStackSecurityGroup +from snaps.openstack.create_user import OpenStackUser +from snaps.openstack.create_volume import OpenStackVolume +from snaps.openstack.create_volume_type import OpenStackVolumeType +from snaps.openstack.os_credentials import OSCreds, ProxySettings +from snaps.openstack.utils import deploy_utils, neutron_utils +from snaps.provisioning import ansible_utils + +logger = logging.getLogger('lanuch_utils') +DEFAULT_CREDS_KEY = 'admin' + + +def launch_config(config, tmplt_file, deploy, clean, clean_image): + """ + Launches all objects and applies any configured ansible playbooks + :param config: the environment configuration dict object + :param tmplt_file: the path to the SNAPS-OO template file + :param deploy: when True deploy + :param clean: when True clean + :param clean_image: when True clean the image when clean is True + """ + os_config = config.get('openstack') + + creators = list() + vm_dict = dict() + images_dict = dict() + flavors_dict = dict() + networks_dict = dict() + routers_dict = dict() + os_creds_dict = dict() + + if os_config: + os_creds_dict = __get_creds_dict(os_config) + + # Create projects + projects_dict = __create_instances( + os_creds_dict, OpenStackProject, ProjectConfig, + os_config.get('projects'), 'project', clean) + creators.append(projects_dict) + + # Create users + users_dict = __create_instances( + os_creds_dict, OpenStackUser, UserConfig, + os_config.get('users'), 'user', clean) + creators.append(users_dict) + + # Associate new users to projects + if not clean: + for project_creator in projects_dict.values(): + users = project_creator.project_settings.users + for user_name in users: + user_creator = users_dict.get(user_name) + if user_creator: + project_creator.assoc_user( + user_creator.get_user()) + + # Create flavors + flavors_dict = __create_instances( + os_creds_dict, OpenStackFlavor, FlavorConfig, + os_config.get('flavors'), 'flavor', clean, users_dict) + creators.append(flavors_dict) + + # Create QoS specs + qos_dict = __create_instances( + os_creds_dict, OpenStackQoS, QoSConfig, + os_config.get('qos_specs'), 'qos_spec', clean, users_dict) + creators.append(qos_dict) + + # Create volume types + vol_type_dict = __create_instances( + os_creds_dict, OpenStackVolumeType, VolumeTypeConfig, + os_config.get('volume_types'), 'volume_type', clean, + users_dict) + creators.append(vol_type_dict) + + # Create volume types + vol_dict = __create_instances( + os_creds_dict, OpenStackVolume, VolumeConfig, + os_config.get('volumes'), 'volume', clean, users_dict) + creators.append(vol_dict) + + # Create images + images_dict = __create_instances( + os_creds_dict, OpenStackImage, ImageConfig, + os_config.get('images'), 'image', clean, users_dict) + creators.append(images_dict) + + # Create networks + networks_dict = __create_instances( + os_creds_dict, OpenStackNetwork, NetworkConfig, + os_config.get('networks'), 'network', clean, users_dict) + creators.append(networks_dict) + + # Create routers + routers_dict = __create_instances( + os_creds_dict, OpenStackRouter, RouterConfig, + os_config.get('routers'), 'router', clean, users_dict) + creators.append(routers_dict) + + # Create keypairs + keypairs_dict = __create_instances( + os_creds_dict, OpenStackKeypair, KeypairConfig, + os_config.get('keypairs'), 'keypair', clean, users_dict) + creators.append(keypairs_dict) + + # Create security groups + creators.append(__create_instances( + os_creds_dict, OpenStackSecurityGroup, + SecurityGroupConfig, + os_config.get('security_groups'), 'security_group', clean, + users_dict)) + + # Create instance + vm_dict = __create_vm_instances( + os_creds_dict, users_dict, os_config.get('instances'), + images_dict, keypairs_dict, clean) + creators.append(vm_dict) + logger.info( + 'Completed creating/retrieving all configured instances') + + # Must enter either block + if clean: + # Cleanup Environment + __cleanup(creators, clean_image) + elif deploy: + # Provision VMs + ansible_config = config.get('ansible') + if ansible_config and vm_dict: + if not __apply_ansible_playbooks( + ansible_config, os_creds_dict, vm_dict, images_dict, + flavors_dict, networks_dict, routers_dict, tmplt_file): + logger.error("Problem applying ansible playbooks") + + +def __get_creds_dict(os_conn_config): + """ + Returns a dict of OSCreds where the key is the creds name. + For backwards compatibility, credentials not contained in a list (only + one) will be returned with the key of None + :param os_conn_config: the credential configuration + :return: a dict of OSCreds objects + """ + if 'connection' in os_conn_config: + return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)} + elif 'connections' in os_conn_config: + out = dict() + for os_conn_dict in os_conn_config['connections']: + config = os_conn_dict.get('connection') + if not config: + raise Exception('Invalid connection format') + + name = config.get('name') + if not name: + raise Exception('Connection config requires a name field') + + out[name] = __get_os_credentials(os_conn_dict) + return out + + +def __get_creds(os_creds_dict, os_user_dict, inst_config): + """ + Returns the appropriate credentials + :param os_creds_dict: a dictionary of OSCreds objects where the name is the + key + :param os_user_dict: a dictionary of OpenStackUser objects where the name + is the key + :param inst_config: + :return: an OSCreds instance or None + """ + os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY) + if 'os_user' in inst_config: + os_user_conf = inst_config['os_user'] + if 'name' in os_user_conf: + user_creator = os_user_dict.get(os_user_conf['name']) + if user_creator: + return user_creator.get_os_creds( + project_name=os_user_conf.get('project_name')) + elif 'os_creds_name' in inst_config: + if 'os_creds_name' in inst_config: + os_creds = os_creds_dict[inst_config['os_creds_name']] + return os_creds + + +def __get_os_credentials(os_conn_config): + """ + Returns an object containing all of the information required to access + OpenStack APIs + :param os_conn_config: The configuration holding the credentials + :return: an OSCreds instance + """ + config = os_conn_config.get('connection') + if not config: + raise Exception('Invalid connection configuration') + + proxy_settings = None + http_proxy = config.get('http_proxy') + if http_proxy: + tokens = re.split(':', http_proxy) + ssh_proxy_cmd = config.get('ssh_proxy_cmd') + proxy_settings = ProxySettings(host=tokens[0], port=tokens[1], + ssh_proxy_cmd=ssh_proxy_cmd) + else: + if 'proxy_settings' in config: + host = config['proxy_settings'].get('host') + port = config['proxy_settings'].get('port') + if host and host != 'None' and port and port != 'None': + proxy_settings = ProxySettings(**config['proxy_settings']) + + if proxy_settings: + config['proxy_settings'] = proxy_settings + else: + if config.get('proxy_settings'): + del config['proxy_settings'] + + return OSCreds(**config) + + +def __parse_ports_config(config): + """ + Parses the "ports" configuration + :param config: The dictionary to parse + :return: a list of PortConfig objects + """ + out = list() + for port_config in config: + out.append(PortConfig(**port_config.get('port'))) + return out + + +def __create_instances(os_creds_dict, creator_class, config_class, config, + config_key, cleanup=False, os_users_dict=None): + """ + Returns a dictionary of SNAPS creator objects where the key is the name + :param os_creds_dict: Dictionary of OSCreds objects where the key is the + name + :param config: The list of configurations for the same type + :param config_key: The list of configurations for the same type + :param cleanup: Denotes whether or not this is being called for cleanup + :return: dictionary + """ + out = {} + + if config: + for config_dict in config: + inst_config = config_dict.get(config_key) + if inst_config: + creds = __get_creds(os_creds_dict, os_users_dict, inst_config) + if creds: + creator = creator_class( + creds, + config_class(**inst_config)) + + if creator: + if cleanup: + try: + creator.initialize() + except Unauthorized as e: + logger.warn( + 'Unable to initialize creator [%s] - %s', + creator, e) + else: + creator.create() + + out[inst_config['name']] = creator + + logger.info('Initialized configured %ss', config_key) + + return out + + +def __create_vm_instances(os_creds_dict, os_users_dict, instances_config, + image_dict, keypairs_dict, cleanup=False): + """ + Returns a dictionary of OpenStackVmInstance objects where the key is the + instance name + :param os_creds_dict: Dictionary of OSCreds objects where the key is the + name + :param os_users_dict: Dictionary of OpenStackUser objects where the key is + the username + :param instances_config: The list of VM instance configurations + :param image_dict: A dictionary of images that will probably be used to + instantiate the VM instance + :param keypairs_dict: A dictionary of keypairs that will probably be used + to instantiate the VM instance + :param cleanup: Denotes whether or not this is being called for cleanup + :return: dictionary + """ + vm_dict = {} + + if instances_config: + for instance_config in instances_config: + conf = instance_config.get('instance') + if conf: + if image_dict: + image_creator = image_dict.get(conf.get('imageName')) + if image_creator: + instance_settings = VmInstanceConfig( + **instance_config['instance']) + kp_creator = keypairs_dict.get( + conf.get('keypair_name')) + + try: + vm_dict[conf[ + 'name']] = deploy_utils.create_vm_instance( + __get_creds( + os_creds_dict, os_users_dict, conf), + instance_settings, + image_creator.image_settings, + keypair_creator=kp_creator, + init_only=cleanup) + except Unauthorized as e: + if not cleanup: + logger.warn('Unable to initialize VM - %s', e) + raise + else: + raise Exception('Image creator instance not found.' + ' Cannot instantiate') + else: + if not cleanup: + raise Exception('Image dictionary is None. Cannot ' + 'instantiate') + else: + raise Exception('Instance configuration is None. Cannot ' + 'instantiate') + logger.info('Created configured instances') + + return vm_dict + + +def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict, + image_dict, flavor_dict, networks_dict, + routers_dict, tmplt_file): + """ + Applies ansible playbooks to running VMs with floating IPs + :param ansible_configs: a list of Ansible configurations + :param os_creds_dict: Dictionary of OSCreds objects where the key is the + name + :param vm_dict: the dictionary of newly instantiated VMs where the name is + the key + :param image_dict: the dictionary of newly instantiated images where the + name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the + name is the key + :param networks_dict: the dictionary of newly instantiated networks where + the name is the key + :param routers_dict: the dictionary of newly instantiated routers where + the name is the key + :param tmplt_file: the path of the SNAPS-OO template file for setting the + CWD so playbook location is relative to the deployment + file + :return: t/f - true if successful + """ + logger.info("Applying Ansible Playbooks") + if ansible_configs: + # Set CWD so the deployment file's playbook location can leverage + # relative paths + orig_cwd = os.getcwd() + env_dir = os.path.dirname(tmplt_file) + os.chdir(env_dir) + + # Apply playbooks + for ansible_config in ansible_configs: + # Ensure all hosts are accepting SSH session requests + for vm_name in ansible_config['hosts']: + vm_inst = vm_dict.get(vm_name) + if vm_inst: + if not vm_inst.vm_ssh_active(block=True): + logger.warning( + 'Timeout waiting for instance to respond to ' + 'SSH requests') + return False + + os_creds = os_creds_dict.get('admin-creds') + __apply_ansible_playbook( + ansible_config, os_creds, vm_dict, image_dict, flavor_dict, + networks_dict, routers_dict) + + # Return to original directory + os.chdir(orig_cwd) + + return True + + +def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, + flavor_dict, networks_dict, routers_dict): + """ + Applies an Ansible configuration setting + :param ansible_config: the configuration settings + :param os_creds: the OpenStack admin credentials object + :param vm_dict: the dictionary of newly instantiated VMs where the name is + the key + :param image_dict: the dictionary of newly instantiated images where the + name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the + name is the key + :param networks_dict: the dictionary of newly instantiated networks where + the name is the key + :param routers_dict: the dictionary of newly instantiated routers where + the name is the key + """ + if ansible_config: + (remote_user, floating_ips, private_key_filepath, + proxy_settings) = __get_connection_info( + ansible_config, vm_dict) + if floating_ips: + for key, vm_creator in vm_dict.items(): + fip = vm_creator.get_floating_ip() + if fip and fip.ip in floating_ips: + if not vm_creator.cloud_init_complete(block=True): + raise Exception( + 'Cannot apply playbooks as cloud-init has not ' + 'completed') + + variables = __get_variables( + ansible_config.get('variables'), os_creds, vm_dict, image_dict, + flavor_dict, networks_dict, routers_dict) + + retval = ansible_utils.apply_playbook( + ansible_config['playbook_location'], floating_ips, remote_user, + private_key_filepath, + variables=variables, + proxy_setting=proxy_settings) + if retval != 0: + # Not a fatal type of event + logger.warning( + 'Unable to apply playbook found at location - %s', + ansible_config.get('playbook_location')) + return retval + + +def __get_connection_info(ansible_config, vm_dict): + """ + Returns a tuple of data required for connecting to the running VMs + (remote_user, [floating_ips], private_key_filepath, proxy_settings) + :param ansible_config: the configuration settings + :param vm_dict: the dictionary of VMs where the VM name is the key + :return: tuple where the first element is the user and the second is a list + of floating IPs and the third is the + private key file location and the fourth is an instance of the + snaps.ProxySettings class + (note: in order to work, each of the hosts need to have the same sudo_user + and private key file location values) + """ + if ansible_config.get('hosts'): + hosts = ansible_config['hosts'] + if len(hosts) > 0: + floating_ips = list() + remote_user = None + pk_file = None + proxy_settings = None + for host in hosts: + vm = vm_dict.get(host) + if vm: + fip = vm.get_floating_ip() + if fip: + remote_user = vm.get_image_user() + + if fip: + floating_ips.append(fip.ip) + else: + raise Exception( + 'Could not find floating IP for VM - ' + + vm.name) + + pk_file = vm.keypair_settings.private_filepath + proxy_settings = vm.get_os_creds().proxy_settings + else: + logger.error('Could not locate VM with name - ' + host) + + return remote_user, floating_ips, pk_file, proxy_settings + return None + + +def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict, + networks_dict, routers_dict): + """ + Returns a dictionary of substitution variables to be used for Ansible + templates + :param var_config: the variable configuration settings + :param os_creds: the OpenStack admin credentials object + :param vm_dict: the dictionary of newly instantiated VMs where the name is + the key + :param image_dict: the dictionary of newly instantiated images where the + name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the + name is the key + :param networks_dict: the dictionary of newly instantiated networks where + the name is the key + :param routers_dict: the dictionary of newly instantiated routers where + the name is the key + :return: dictionary or None + """ + if var_config and vm_dict and len(vm_dict) > 0: + variables = dict() + for key, value in var_config.items(): + value = __get_variable_value( + value, os_creds, vm_dict, image_dict, flavor_dict, + networks_dict, routers_dict) + if key and value: + variables[key] = value + logger.info( + "Set Jinga2 variable with key [%s] the value [%s]", + key, value) + else: + raise Exception( + 'Key - [' + str(key) + '] or Value [' + str(value) + + '] must not be None') + return variables + return None + + +def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict, + flavor_dict, networks_dict, routers_dict): + """ + Returns the associated variable value for use by Ansible for substitution + purposes + :param var_config_values: the configuration dictionary + :param os_creds: the OpenStack admin credentials object + :param vm_dict: the dictionary of newly instantiated VMs where the name is + the key + :param image_dict: the dictionary of newly instantiated images where the + name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the + name is the key + :param networks_dict: the dictionary of newly instantiated networks where + the name is the key + :param routers_dict: the dictionary of newly instantiated routers where + the name is the key + :return: + """ + if var_config_values['type'] == 'string': + return __get_string_variable_value(var_config_values) + if var_config_values['type'] == 'vm-attr': + return __get_vm_attr_variable_value(var_config_values, vm_dict) + if var_config_values['type'] == 'os_creds': + return __get_os_creds_variable_value(var_config_values, os_creds) + if var_config_values['type'] == 'network': + return __get_network_variable_value(var_config_values, networks_dict) + if var_config_values['type'] == 'router': + return __get_router_variable_value(var_config_values, routers_dict, + os_creds) + if var_config_values['type'] == 'port': + return __get_vm_port_variable_value(var_config_values, vm_dict) + if var_config_values['type'] == 'floating_ip': + return __get_vm_fip_variable_value(var_config_values, vm_dict) + if var_config_values['type'] == 'image': + return __get_image_variable_value(var_config_values, image_dict) + if var_config_values['type'] == 'flavor': + return __get_flavor_variable_value(var_config_values, flavor_dict) + return None + + +def __get_string_variable_value(var_config_values): + """ + Returns the associated string value + :param var_config_values: the configuration dictionary + :return: the value contained in the dictionary with the key 'value' + """ + return var_config_values['value'] + + +def __get_vm_attr_variable_value(var_config_values, vm_dict): + """ + Returns the associated value contained on a VM instance + :param var_config_values: the configuration dictionary + :param vm_dict: the dictionary containing all VMs where the key is the VM's + name + :return: the value + """ + vm = vm_dict.get(var_config_values['vm_name']) + if vm: + if var_config_values['value'] == 'floating_ip': + return vm.get_floating_ip().ip + if var_config_values['value'] == 'image_user': + return vm.get_image_user() + + +def __get_os_creds_variable_value(var_config_values, os_creds): + """ + Returns the associated OS credentials value + :param var_config_values: the configuration dictionary + :param os_creds: the admin OpenStack OSCreds object + :return: the value + """ + if os_creds: + if var_config_values['value'] == 'username': + logger.info("Returning OS username") + return os_creds.username + elif var_config_values['value'] == 'password': + logger.info("Returning OS password") + return os_creds.password + elif var_config_values['value'] == 'auth_url': + logger.info("Returning OS auth_url") + return os_creds.auth_url + elif var_config_values['value'] == 'project_name': + logger.info("Returning OS project_name") + return os_creds.project_name + + +def __get_network_variable_value(var_config_values, networks_dict): + """ + Returns the associated network value + :param var_config_values: the configuration dictionary + :param networks_dict: the dictionary containing all networks where the key + is the network name + :return: the value + """ + net_name = var_config_values.get('network_name') + + if net_name and networks_dict.get(net_name): + network_creator = networks_dict[net_name] + + if 'subnet_name' in var_config_values: + subnet_name = var_config_values.get('subnet_name') + if subnet_name: + for subnet in network_creator.get_network().subnets: + if subnet_name == subnet.name: + if 'value' in var_config_values: + if 'gateway_ip' == var_config_values['value']: + return subnet.gateway_ip + if 'ip_range' == var_config_values['value']: + return subnet.start + ' ' + subnet.end + if 'cidr_ip' == var_config_values['value']: + cidr_split = subnet.cidr.split('/') + return cidr_split[0] + if 'netmask' == var_config_values['value']: + cidr_split = subnet.cidr.split('/') + cidr_bits = 32 - int(cidr_split[1]) + netmask = socket.inet_ntoa( + struct.pack( + '!I', (1 << 32) - (1 << cidr_bits))) + return netmask + if 'broadcast_ip' == var_config_values['value']: + end_split = subnet.end.split('.') + broadcast_ip = ( + end_split[0] + '.' + end_split[1] + '.' + + end_split[2] + '.255') + return broadcast_ip + + +def __get_router_variable_value(var_config_values, routers_dict, os_creds): + """ + Returns the associated network value + :param var_config_values: the configuration dictionary + :param routers_dict: the dictionary containing all networks where the key + is the network name + :param os_creds: the admin OpenStack credentials + :return: the value + """ + router_name = var_config_values.get('router_name') + router_creator = routers_dict[router_name] + + if router_creator: + if 'external_fixed_ip' == var_config_values.get('attr'): + neutron = neutron_utils.neutron_client(os_creds) + ext_nets = neutron_utils.get_external_networks(neutron) + + subnet_name = var_config_values.get('subnet_name') + + for ext_net in ext_nets: + for subnet in ext_net.subnets: + if subnet_name == subnet.name: + router = router_creator.get_router() + for fixed_ips in router.external_fixed_ips: + if subnet.id == fixed_ips['subnet_id']: + return fixed_ips['ip_address'] + + +def __get_vm_port_variable_value(var_config_values, vm_dict): + """ + Returns the associated OS credentials value + :param var_config_values: the configuration dictionary + :param vm_dict: the dictionary containing all VMs where the key is the VM's + name + :return: the value + """ + port_name = var_config_values.get('port_name') + vm_name = var_config_values.get('vm_name') + + if port_name and vm_name: + vm = vm_dict.get(vm_name) + if vm: + for vm_port in vm.get_vm_inst().ports: + if vm_port.name == port_name: + port_value_id = var_config_values.get('port_value') + if port_value_id: + if port_value_id == 'mac_address': + return vm_port.mac_address + if port_value_id == 'ip_address': + return vm_port.ips[0]['ip_address'] + + +def __get_vm_fip_variable_value(var_config_values, vm_dict): + """ + Returns the floating IP value if found + :param var_config_values: the configuration dictionary + :param vm_dict: the dictionary containing all VMs where the key is the VM's + name + :return: the floating IP string value or None + """ + fip_name = var_config_values.get('fip_name') + vm_name = var_config_values.get('vm_name') + + if vm_name: + vm = vm_dict.get(vm_name) + if vm: + fip = vm.get_floating_ip(fip_name) + if fip: + return fip.ip + + +def __get_image_variable_value(var_config_values, image_dict): + """ + Returns the associated image value + :param var_config_values: the configuration dictionary + :param image_dict: the dictionary containing all images where the key is + the name + :return: the value + """ + if image_dict: + if var_config_values.get('image_name'): + image_creator = image_dict.get(var_config_values['image_name']) + if image_creator: + if (var_config_values.get('value') + and var_config_values['value'] == 'id'): + return image_creator.get_image().id + if (var_config_values.get('value') + and var_config_values['value'] == 'user'): + return image_creator.image_settings.image_user + + +def __get_flavor_variable_value(var_config_values, flavor_dict): + """ + Returns the associated flavor value + :param var_config_values: the configuration dictionary + :param flavor_dict: the dictionary containing all flavor creators where the + key is the name + :return: the value or None + """ + if flavor_dict: + if var_config_values.get('flavor_name'): + flavor_creator = flavor_dict.get(var_config_values['flavor_name']) + if flavor_creator: + if (var_config_values.get('value') + and var_config_values['value'] == 'id'): + return flavor_creator.get_flavor().id + + +def __cleanup(creators, clean_image=False): + """ + Cleans up environment + :param creators: the list of creators by type + :param clean_image: when true + :return: + """ + for creator_dict in reversed(creators): + for key, creator in creator_dict.items(): + if ((isinstance(creator, OpenStackImage) and clean_image) + or not isinstance(creator, OpenStackImage)): + creator.clean() diff --git a/snaps/provisioning/ansible_pb/__init__.py b/snaps/provisioning/ansible_pb/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/snaps/provisioning/ansible_pb/__init__.py +++ /dev/null diff --git a/snaps/provisioning/ansible_pb/centos-network-setup/__init__.py b/snaps/provisioning/ansible_pb/centos-network-setup/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/snaps/provisioning/ansible_pb/centos-network-setup/__init__.py +++ /dev/null diff --git a/snaps/provisioning/ansible_pb/centos-network-setup/playbooks/__init__.py b/snaps/provisioning/ansible_pb/centos-network-setup/playbooks/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/snaps/provisioning/ansible_pb/centos-network-setup/playbooks/__init__.py +++ /dev/null diff --git a/snaps/provisioning/ansible_pb/centos-network-setup/playbooks/configure_host.yml b/snaps/provisioning/ansible_pb/centos-network-setup/playbooks/configure_host.yml deleted file mode 100644 index 8df03cb..0000000 --- a/snaps/provisioning/ansible_pb/centos-network-setup/playbooks/configure_host.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") -# and others. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Configure NIC - hosts: all - become: yes - become_method: sudo - become_user: root - - tasks: - - name: Setup /etc/sysconfig/network-scripts/ifcfg-eth1 file - action: template owner=root group=root mode=644 src=../templates/ifcfg-interface dest=/etc/sysconfig/network-scripts/ifcfg-{{nic_name}} - - name : Restart Network - command: systemctl restart network
\ No newline at end of file diff --git a/snaps/provisioning/ansible_pb/centos-network-setup/templates/ifcfg-interface b/snaps/provisioning/ansible_pb/centos-network-setup/templates/ifcfg-interface deleted file mode 100644 index 47aa3fa..0000000 --- a/snaps/provisioning/ansible_pb/centos-network-setup/templates/ifcfg-interface +++ /dev/null @@ -1,14 +0,0 @@ -DEVICE={{ nic_name }} -NAME={{ nic_name }} -IPADDR={{ nic_ip }} - -DEFROUTE=no -NETMASK=255.255.255.0 -NM_CONTROLLED=no -IPV6INIT=yes -IPV6_AUTOCONF=yes -IPV6_DEFROUTE=yes -IPV6_PEERDNS=yes -IPV6_PEERROUTES=yes -IPV6_FAILURE_FATAL=no -ONBOOT=yes
\ No newline at end of file diff --git a/snaps/provisioning/ansible_pb/ubuntu-network-setup/__init__.py b/snaps/provisioning/ansible_pb/ubuntu-network-setup/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/snaps/provisioning/ansible_pb/ubuntu-network-setup/__init__.py +++ /dev/null diff --git a/snaps/provisioning/ansible_pb/ubuntu-network-setup/playbooks/__init__.py b/snaps/provisioning/ansible_pb/ubuntu-network-setup/playbooks/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/snaps/provisioning/ansible_pb/ubuntu-network-setup/playbooks/__init__.py +++ /dev/null diff --git a/snaps/provisioning/ansible_pb/ubuntu-network-setup/playbooks/configure_host.yml b/snaps/provisioning/ansible_pb/ubuntu-network-setup/playbooks/configure_host.yml deleted file mode 100644 index 5d43f96..0000000 --- a/snaps/provisioning/ansible_pb/ubuntu-network-setup/playbooks/configure_host.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") -# and others. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Configure NIC - hosts: all - become: yes - become_method: sudo - become_user: root - - tasks: - - name: Setup /etc/network/interfaces.d/{{nic_name}}.cfg file - action: template owner=root group=root mode=644 src=../templates/ethN.cfg dest=/etc/network/interfaces.d/{{nic_name}}.cfg - - name : Restart Network - command: service networking restart
\ No newline at end of file diff --git a/snaps/provisioning/ansible_pb/ubuntu-network-setup/templates/ethN.cfg b/snaps/provisioning/ansible_pb/ubuntu-network-setup/templates/ethN.cfg deleted file mode 100644 index 3fa7708..0000000 --- a/snaps/provisioning/ansible_pb/ubuntu-network-setup/templates/ethN.cfg +++ /dev/null @@ -1,2 +0,0 @@ -auto {{ nic_name }} -iface {{ nic_name }} inet dhcp diff --git a/snaps/provisioning/tests/ansible_utils_tests.py b/snaps/provisioning/tests/ansible_utils_tests.py index 4f1f65e..7600002 100644 --- a/snaps/provisioning/tests/ansible_utils_tests.py +++ b/snaps/provisioning/tests/ansible_utils_tests.py @@ -241,6 +241,9 @@ class AnsibleProvisioningTests(OSIntegrationTestCase): # Block until VM's ssh port has been opened self.assertTrue(self.inst_creator.vm_ssh_active(block=True)) + # Block until cloud-init has completed + self.assertTrue(self.inst_creator.cloud_init_complete(block=True)) + ssh_client = self.inst_creator.ssh_client() self.assertIsNotNone(ssh_client) @@ -310,6 +313,9 @@ class AnsibleProvisioningTests(OSIntegrationTestCase): # Block until VM's ssh port has been opened self.assertTrue(self.inst_creator.vm_ssh_active(block=True)) + # Block until cloud-init has completed + self.assertTrue(self.inst_creator.cloud_init_complete(block=True)) + # Apply Security Group self.inst_creator.add_security_group( self.sec_grp_creator.get_security_group()) diff --git a/snaps/test_suite_builder.py b/snaps/test_suite_builder.py index ec8196d..1990cd6 100644 --- a/snaps/test_suite_builder.py +++ b/snaps/test_suite_builder.py @@ -65,13 +65,13 @@ from snaps.openstack.tests.create_image_tests import ( CreateImageSuccessTests, CreateImageNegativeTests, CreateMultiPartImageTests) from snaps.openstack.tests.create_instance_tests import ( - CreateInstanceSingleNetworkTests, CreateInstancePubPrivNetTests, - CreateInstanceOnComputeHost, CreateInstanceSimpleTests, - FloatingIpSettingsUnitTests, InstanceSecurityGroupTests, - VmInstanceSettingsUnitTests, CreateInstancePortManipulationTests, - SimpleHealthCheck, CreateInstanceFromThreePartImage, - CreateInstanceMockOfflineTests, CreateInstanceTwoNetTests, - CreateInstanceVolumeTests, CreateInstanceIPv6NetworkTests) + CreateInstanceSingleNetworkTests, CreateInstanceOnComputeHost, + CreateInstanceSimpleTests, FloatingIpSettingsUnitTests, + InstanceSecurityGroupTests, VmInstanceSettingsUnitTests, + CreateInstancePortManipulationTests, SimpleHealthCheck, + CreateInstanceFromThreePartImage, CreateInstanceMockOfflineTests, + CreateInstanceTwoNetTests, CreateInstanceVolumeTests, + CreateInstanceIPv6NetworkTests) from snaps.openstack.tests.create_keypairs_tests import ( CreateKeypairsTests, KeypairSettingsUnitTests, CreateKeypairsCleanupTests) from snaps.openstack.tests.create_network_tests import ( @@ -638,11 +638,6 @@ def add_openstack_integration_tests(suite, os_creds, ext_net_name, flavor_metadata=flavor_metadata, image_metadata=image_metadata, log_level=log_level)) suite.addTest(OSIntegrationTestCase.parameterize( - CreateInstancePubPrivNetTests, os_creds=os_creds, - ext_net_name=ext_net_name, use_keystone=use_keystone, - flavor_metadata=flavor_metadata, image_metadata=image_metadata, - log_level=log_level)) - suite.addTest(OSIntegrationTestCase.parameterize( AnsibleProvisioningTests, os_creds=os_creds, ext_net_name=ext_net_name, use_keystone=use_keystone, flavor_metadata=flavor_metadata, image_metadata=image_metadata, |