diff options
| -rw-r--r-- | docs/how-to-use/VirtEnvDeploy.rst | 4 | ||||
| -rw-r--r-- | examples/launch.py | 413 | ||||
| -rw-r--r-- | snaps/file_utils.py | 7 | ||||
| -rw-r--r-- | snaps/openstack/create_image.py | 13 | ||||
| -rw-r--r-- | snaps/openstack/create_instance.py | 8 | ||||
| -rw-r--r-- | snaps/openstack/create_keypairs.py | 20 | ||||
| -rw-r--r-- | snaps/openstack/create_project.py | 27 | ||||
| -rw-r--r-- | snaps/openstack/create_security_group.py | 13 | ||||
| -rw-r--r-- | snaps/openstack/os_credentials.py | 2 | ||||
| -rw-r--r-- | snaps/openstack/tests/conf/os_credentials_tests.py | 16 | ||||
| -rw-r--r-- | snaps/openstack/tests/create_image_tests.py | 13 | ||||
| -rw-r--r-- | snaps/openstack/tests/create_project_tests.py | 13 | ||||
| -rw-r--r-- | snaps/openstack/utils/glance_utils.py | 4 | ||||
| -rw-r--r-- | snaps/openstack/utils/keystone_utils.py | 7 | ||||
| -rw-r--r-- | snaps/openstack/utils/nova_utils.py | 67 |
15 files changed, 382 insertions, 245 deletions
diff --git a/docs/how-to-use/VirtEnvDeploy.rst b/docs/how-to-use/VirtEnvDeploy.rst index f8a1e3c..dd95202 100644 --- a/docs/how-to-use/VirtEnvDeploy.rst +++ b/docs/how-to-use/VirtEnvDeploy.rst @@ -29,13 +29,13 @@ Use launcher.py to deploy and clean up example environments. These examples are :: - python launch.py -e ./complex-network/deploy-complex-network.yaml -d + python launch.py -t ./complex-network/deploy-complex-network.yaml -d #. Clean the deployment. :: - python launch.py -e ./complex-network/deploy-complex-network.yaml -c + python launch.py -t ./complex-network/deploy-complex-network.yaml -c #. Customize the deployment by changing the yaml file. diff --git a/examples/launch.py b/examples/launch.py index 65142ef..f5d3bea 100644 --- a/examples/launch.py +++ b/examples/launch.py @@ -20,23 +20,82 @@ import argparse import logging import re +import time +from jinja2 import Environment, FileSystemLoader import os +import yaml + from snaps import file_utils from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor from snaps.openstack.create_image import ImageSettings, OpenStackImage from snaps.openstack.create_instance import VmInstanceSettings -from snaps.openstack.create_keypairs import KeypairSettings -from snaps.openstack.create_network import PortSettings, NetworkSettings -from snaps.openstack.create_router import RouterSettings +from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair +from snaps.openstack.create_network import ( + PortSettings, NetworkSettings, OpenStackNetwork) +from snaps.openstack.create_project import OpenStackProject, ProjectSettings +from snaps.openstack.create_router import RouterSettings, OpenStackRouter +from snaps.openstack.create_security_group import ( + OpenStackSecurityGroup, SecurityGroupSettings) +from snaps.openstack.create_user import OpenStackUser, UserSettings from snaps.openstack.os_credentials import OSCreds, ProxySettings from snaps.openstack.utils import deploy_utils from snaps.provisioning import ansible_utils __author__ = 'spisarski' -logger = logging.getLogger('deploy_venv') +logger = logging.getLogger('snaps_launcher') ARG_NOT_SET = "argument not set" +DEFAULT_CREDS_KEY = 'admin' + + +def __get_creds_dict(os_conn_config): + """ + Returns a dict of OSCreds where the key is the creds name. + For backwards compatibility, credentials not contained in a list (only + one) will be returned with the key of None + :param os_conn_config: the credential configuration + :return: a dict of OSCreds objects + """ + if 'connection' in os_conn_config: + return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)} + elif 'connections' in os_conn_config: + out = dict() + for os_conn_dict in os_conn_config['connections']: + config = os_conn_dict.get('connection') + if not config: + raise Exception('Invalid connection format') + + name = config.get('name') + if not name: + raise Exception('Connection config requires a name field') + + out[name] = __get_os_credentials(os_conn_dict) + return out + + +def __get_creds(os_creds_dict, os_user_dict, inst_config): + """ + Returns the appropriate credentials + :param os_creds_dict: a dictionary of OSCreds objects where the name is the + key + :param os_user_dict: a dictionary of OpenStackUser objects where the name + is the key + :param inst_config: + :return: an OSCreds instance or None + """ + os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY) + if 'os_user' in inst_config: + os_user_conf = inst_config['os_user'] + if 'name' in os_user_conf: + user_creator = os_user_dict.get(os_user_conf['name']) + if user_creator: + return user_creator.get_os_creds( + project_name=os_user_conf.get('project_name')) + elif 'os_creds_name' in inst_config: + if 'os_creds_name' in inst_config: + os_creds = os_creds_dict[inst_config['os_creds_name']] + return os_creds def __get_os_credentials(os_conn_config): @@ -46,17 +105,30 @@ def __get_os_credentials(os_conn_config): :param os_conn_config: The configuration holding the credentials :return: an OSCreds instance """ + config = os_conn_config.get('connection') + if not config: + raise Exception('Invalid connection configuration') + proxy_settings = None - http_proxy = os_conn_config.get('http_proxy') + http_proxy = config.get('http_proxy') if http_proxy: tokens = re.split(':', http_proxy) - ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd') + ssh_proxy_cmd = config.get('ssh_proxy_cmd') proxy_settings = ProxySettings(host=tokens[0], port=tokens[1], ssh_proxy_cmd=ssh_proxy_cmd) + else: + if 'proxy_settings' in config: + host = config['proxy_settings'].get('host') + port = config['proxy_settings'].get('port') + if host and host != 'None' and port and port != 'None': + proxy_settings = ProxySettings(**config['proxy_settings']) + + if proxy_settings: + config['proxy_settings'] = proxy_settings + else: + del config['proxy_settings'] - os_conn_config['proxy_settings'] = proxy_settings - - return OSCreds(**os_conn_config) + return OSCreds(**config) def __parse_ports_config(config): @@ -71,155 +143,46 @@ def __parse_ports_config(config): return out -def __create_flavors(os_conn_config, flavors_config, cleanup=False): - """ - Returns a dictionary of flavors where the key is the image name and the - value is the image object - :param os_conn_config: The OpenStack connection credentials - :param flavors_config: The list of image configurations - :param cleanup: Denotes whether or not this is being called for cleanup - :return: dictionary - """ - flavors = {} - - if flavors_config: - try: - for flavor_config_dict in flavors_config: - flavor_config = flavor_config_dict.get('flavor') - if flavor_config and flavor_config.get('name'): - flavor_creator = OpenStackFlavor( - __get_os_credentials(os_conn_config), - FlavorSettings(**flavor_config)) - flavor_creator.create(cleanup=cleanup) - flavors[flavor_config['name']] = flavor_creator - except Exception as e: - for key, flavor_creator in flavors.items(): - flavor_creator.clean() - raise e - logger.info('Created configured flavors') - - return flavors - - -def __create_images(os_conn_config, images_config, cleanup=False): - """ - Returns a dictionary of images where the key is the image name and the - value is the image object - :param os_conn_config: The OpenStack connection credentials - :param images_config: The list of image configurations - :param cleanup: Denotes whether or not this is being called for cleanup - :return: dictionary - """ - images = {} - - if images_config: - try: - for image_config_dict in images_config: - image_config = image_config_dict.get('image') - if image_config and image_config.get('name'): - images[image_config['name']] = deploy_utils.create_image( - __get_os_credentials(os_conn_config), - ImageSettings(**image_config), cleanup) - except Exception as e: - for key, image_creator in images.items(): - image_creator.clean() - raise e - logger.info('Created configured images') - - return images - - -def __create_networks(os_conn_config, network_confs, cleanup=False): - """ - Returns a dictionary of networks where the key is the network name and the - value is the network object - :param os_conn_config: The OpenStack connection credentials - :param network_confs: The list of network configurations - :param cleanup: Denotes whether or not this is being called for cleanup - :return: dictionary - """ - network_dict = {} - - if network_confs: - try: - for network_conf in network_confs: - net_name = network_conf['network']['name'] - os_creds = __get_os_credentials(os_conn_config) - network_dict[net_name] = deploy_utils.create_network( - os_creds, NetworkSettings(**network_conf['network']), - cleanup) - except Exception as e: - for key, net_creator in network_dict.items(): - net_creator.clean() - raise e - - logger.info('Created configured networks') - - return network_dict - - -def __create_routers(os_conn_config, router_confs, cleanup=False): +def __create_instances(os_creds_dict, creator_class, config_class, config, + config_key, cleanup=False, os_users_dict=None): """ - Returns a dictionary of networks where the key is the network name and the - value is the network object - :param os_conn_config: The OpenStack connection credentials - :param router_confs: The list of router configurations + Returns a dictionary of SNAPS creator objects where the key is the name + :param os_creds_dict: Dictionary of OSCreds objects where the key is the + name + :param config: The list of configurations for the same type + :param config_key: The list of configurations for the same type :param cleanup: Denotes whether or not this is being called for cleanup :return: dictionary """ - router_dict = {} - os_creds = __get_os_credentials(os_conn_config) + out = {} - if router_confs: - try: - for router_conf in router_confs: - router_name = router_conf['router']['name'] - router_dict[router_name] = deploy_utils.create_router( - os_creds, RouterSettings(**router_conf['router']), cleanup) - except Exception as e: - for key, router_creator in router_dict.items(): - router_creator.clean() - raise e - - logger.info('Created configured networks') - - return router_dict - - -def __create_keypairs(os_conn_config, keypair_confs, cleanup=False): - """ - Returns a dictionary of keypairs where the key is the keypair name and the - value is the keypair object - :param os_conn_config: The OpenStack connection credentials - :param keypair_confs: The list of keypair configurations - :param cleanup: Denotes whether or not this is being called for cleanup - :return: dictionary - """ - keypairs_dict = {} - if keypair_confs: + if config: try: - for keypair_dict in keypair_confs: - keypair_config = keypair_dict['keypair'] - kp_settings = KeypairSettings(**keypair_config) - keypairs_dict[ - keypair_config['name']] = deploy_utils.create_keypair( - __get_os_credentials(os_conn_config), kp_settings, cleanup) + for config_dict in config: + inst_config = config_dict.get(config_key) + if inst_config: + creator = creator_class( + __get_creds(os_creds_dict, os_users_dict, inst_config), + config_class(**inst_config)) + creator.create(cleanup=cleanup) + out[inst_config['name']] = creator + logger.info('Created configured %s', config_key) except Exception as e: - for key, keypair_creator in keypairs_dict.items(): - keypair_creator.clean() - raise e - - logger.info('Created configured keypairs') + logger.error('Unexpected error instantiating creator [%s] ' + 'with exception %s', creator_class, e) - return keypairs_dict + return out -def __create_instances(os_conn_config, instances_config, image_dict, - keypairs_dict, cleanup=False): +def __create_vm_instances(os_creds_dict, os_users_dict, instances_config, + image_dict, keypairs_dict, cleanup=False): """ - Returns a dictionary of instances where the key is the instance name and - the value is the VM object - :param os_conn_config: The OpenStack connection credentials + Returns a dictionary of OpenStackVmInstance objects where the key is the + instance name + :param os_creds_dict: Dictionary of OSCreds objects where the key is the + name + :param os_users_dict: Dictionary of OpenStackUser objects where the key is + the username :param instances_config: The list of VM instance configurations :param image_dict: A dictionary of images that will probably be used to instantiate the VM instance @@ -228,8 +191,6 @@ def __create_instances(os_conn_config, instances_config, image_dict, :param cleanup: Denotes whether or not this is being called for cleanup :return: dictionary """ - os_creds = __get_os_credentials(os_conn_config) - vm_dict = {} if instances_config: @@ -245,7 +206,9 @@ def __create_instances(os_conn_config, instances_config, image_dict, kp_name = conf.get('keypair_name') vm_dict[conf[ 'name']] = deploy_utils.create_vm_instance( - os_creds, instance_settings, + __get_creds( + os_creds_dict, os_users_dict, conf), + instance_settings, image_creator.image_settings, keypair_creator=keypairs_dict[kp_name], cleanup=cleanup) @@ -258,24 +221,19 @@ def __create_instances(os_conn_config, instances_config, image_dict, else: raise Exception('Instance configuration is None. Cannot ' 'instantiate') + logger.info('Created configured instances') except Exception as e: - logger.error('Unexpected error creating instances. Attempting to ' - 'cleanup environment - %s', e) - for key, inst_creator in vm_dict.items(): - inst_creator.clean() - raise e - - logger.info('Created configured instances') + logger.error('Unexpected error creating VM instances - %s', e) return vm_dict -def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict, +def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict, image_dict, flavor_dict, env_file): """ Applies ansible playbooks to running VMs with floating IPs :param ansible_configs: a list of Ansible configurations - :param os_conn_config: the OpenStack connection configuration used to - create an OSCreds instance + :param os_creds_dict: Dictionary of OSCreds objects where the key is the + name :param vm_dict: the dictionary of newly instantiated VMs where the name is the key :param image_dict: the dictionary of newly instantiated images where the @@ -303,7 +261,16 @@ def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict, # Apply playbooks for ansible_config in ansible_configs: - os_creds = __get_os_credentials(os_conn_config) + if 'pre_sleep_time' in ansible_config: + try: + sleep_time = int(ansible_config['pre_sleep_time']) + logger.info('Waiting %s seconds to apply playbooks', + sleep_time) + time.sleep(sleep_time) + except: + pass + + os_creds = os_creds_dict.get(None, 'admin') __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict) @@ -341,8 +308,8 @@ def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, if retval != 0: # Not a fatal type of event logger.warning( - 'Unable to apply playbook found at location - ' + - ansible_config('playbook_location')) + 'Unable to apply playbook found at location - %s', + ansible_config.get('playbook_location')) def __get_connection_info(ansible_config, vm_dict): @@ -442,6 +409,8 @@ def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict, return __get_os_creds_variable_value(var_config_values, os_creds) if var_config_values['type'] == 'port': return __get_vm_port_variable_value(var_config_values, vm_dict) + if var_config_values['type'] == 'floating_ip': + return __get_vm_fip_variable_value(var_config_values, vm_dict) if var_config_values['type'] == 'image': return __get_image_variable_value(var_config_values, image_dict) if var_config_values['type'] == 'flavor': @@ -522,6 +491,25 @@ def __get_vm_port_variable_value(var_config_values, vm_dict): return vm.get_port_ip(port_name) +def __get_vm_fip_variable_value(var_config_values, vm_dict): + """ + Returns the floating IP value if found + :param var_config_values: the configuration dictionary + :param vm_dict: the dictionary containing all VMs where the key is the VM's + name + :return: the floating IP string value or None + """ + fip_name = var_config_values.get('fip_name') + vm_name = var_config_values.get('vm_name') + + if vm_name: + vm = vm_dict.get(vm_name) + if vm: + fip = vm.get_floating_ip(fip_name) + if fip: + return fip.ip + + def __get_image_variable_value(var_config_values, image_dict): """ Returns the associated image value @@ -585,53 +573,93 @@ def main(arguments): logging.basicConfig(level=log_level) logger.info('Starting to Deploy') - config = file_utils.read_yaml(arguments.environment) - logger.debug('Read configuration file - ' + arguments.environment) + + # Apply env_file/substitution file to template + env = Environment(loader=FileSystemLoader( + searchpath=os.path.dirname(arguments.tmplt_file))) + template = env.get_template(os.path.basename(arguments.tmplt_file)) + + env_dict = dict() + if arguments.env_file: + env_dict = file_utils.read_yaml(arguments.env_file) + output = template.render(**env_dict) + + config = yaml.load(output) if config: os_config = config.get('openstack') - os_conn_config = None creators = list() vm_dict = dict() images_dict = dict() flavors_dict = dict() + os_creds_dict = dict() + clean = arguments.clean is not ARG_NOT_SET if os_config: + os_creds_dict = __get_creds_dict(os_config) + try: - os_conn_config = os_config.get('connection') + # Create projects + projects_dict = __create_instances( + os_creds_dict, OpenStackProject, ProjectSettings, + os_config.get('projects'), 'project', clean) + creators.append(projects_dict) + + # Create users + users_dict = __create_instances( + os_creds_dict, OpenStackUser, UserSettings, + os_config.get('users'), 'user', clean) + creators.append(users_dict) + + # Associate new users to projects + if not clean: + for project_creator in projects_dict.values(): + users = project_creator.project_settings.users + for user_name in users: + user_creator = users_dict.get(user_name) + if user_creator: + project_creator.assoc_user( + user_creator.get_user()) # Create flavors - flavors_dict = __create_flavors( - os_conn_config, os_config.get('flavors'), - arguments.clean is not ARG_NOT_SET) + flavors_dict = __create_instances( + os_creds_dict, OpenStackFlavor, FlavorSettings, + os_config.get('flavors'), 'flavor', clean, users_dict) creators.append(flavors_dict) # Create images - images_dict = __create_images( - os_conn_config, os_config.get('images'), - arguments.clean is not ARG_NOT_SET) + images_dict = __create_instances( + os_creds_dict, OpenStackImage, ImageSettings, + os_config.get('images'), 'image', clean, users_dict) creators.append(images_dict) - # Create network - creators.append(__create_networks( - os_conn_config, os_config.get('networks'), - arguments.clean is not ARG_NOT_SET)) + # Create networks + creators.append(__create_instances( + os_creds_dict, OpenStackNetwork, NetworkSettings, + os_config.get('networks'), 'network', clean, users_dict)) # Create routers - creators.append(__create_routers( - os_conn_config, os_config.get('routers'), - arguments.clean is not ARG_NOT_SET)) + creators.append(__create_instances( + os_creds_dict, OpenStackRouter, RouterSettings, + os_config.get('routers'), 'router', clean, users_dict)) # Create keypairs - keypairs_dict = __create_keypairs( - os_conn_config, os_config.get('keypairs'), - arguments.clean is not ARG_NOT_SET) + keypairs_dict = __create_instances( + os_creds_dict, OpenStackKeypair, KeypairSettings, + os_config.get('keypairs'), 'keypair', clean, users_dict) creators.append(keypairs_dict) + # Create security groups + creators.append(__create_instances( + os_creds_dict, OpenStackSecurityGroup, + SecurityGroupSettings, + os_config.get('security_groups'), 'security_group', clean, + users_dict)) + # Create instance - vm_dict = __create_instances( - os_conn_config, os_config.get('instances'), + vm_dict = __create_vm_instances( + os_creds_dict, users_dict, os_config.get('instances'), images_dict, keypairs_dict, arguments.clean is not ARG_NOT_SET) creators.append(vm_dict) @@ -641,7 +669,7 @@ def main(arguments): logger.error( 'Unexpected error deploying environment. Rolling back due' ' to - ' + str(e)) - __cleanup(creators) + # __cleanup(creators) raise # Must enter either block @@ -658,13 +686,13 @@ def main(arguments): ansible_config = config.get('ansible') if ansible_config and vm_dict: if not __apply_ansible_playbooks(ansible_config, - os_conn_config, vm_dict, + os_creds_dict, vm_dict, images_dict, flavors_dict, - arguments.environment): + arguments.tmplt_file): logger.error("Problem applying ansible playbooks") else: logger.error( - 'Unable to read configuration file - ' + arguments.environment) + 'Unable to read configuration file - ' + arguments.tmplt_file) exit(1) exit(0) @@ -698,8 +726,11 @@ if __name__ == '__main__': default=ARG_NOT_SET, help='When cleaning, if this is set, the image will be cleaned too') parser.add_argument( - '-e', '--env', dest='environment', required=True, - help='The environment configuration YAML file - REQUIRED') + '-t', '--tmplt', dest='tmplt_file', required=True, + help='The SNAPS deployment template YAML file - REQUIRED') + parser.add_argument( + '-e', '--env-file', dest='env_file', + help='Yaml file containing substitution values to the env file') parser.add_argument( '-l', '--log-level', dest='log_level', default='INFO', help='Logging Level (INFO|DEBUG)') diff --git a/snaps/file_utils.py b/snaps/file_utils.py index 699d378..a421dd3 100644 --- a/snaps/file_utils.py +++ b/snaps/file_utils.py @@ -39,10 +39,11 @@ def file_exists(file_path): the path is a directory :return: """ - if os.path.exists(file_path): - if os.path.isdir(file_path): + expanded_path = os.path.expanduser(file_path) + if os.path.exists(expanded_path): + if os.path.isdir(expanded_path): return False - return os.path.isfile(file_path) + return os.path.isfile(expanded_path) return False diff --git a/snaps/openstack/create_image.py b/snaps/openstack/create_image.py index a4c9357..537824d 100644 --- a/snaps/openstack/create_image.py +++ b/snaps/openstack/create_image.py @@ -61,8 +61,8 @@ class OpenStackImage: if self.__image: logger.info('Found image with name - ' + self.image_settings.name) return self.__image - elif self.image_settings.exists and not self.image_settings.url \ - and not self.image_settings.image_file: + elif (self.image_settings.exists and not self.image_settings.url + and not self.image_settings.image_file): raise ImageCreationError( 'Image with does not exist with name - ' + self.image_settings.name) @@ -257,8 +257,13 @@ class ImageSettings: self.url = kwargs.get('url') if not self.url: self.url = kwargs.get('download_url') + if self.url == 'None': + self.url = None self.image_file = kwargs.get('image_file') + if self.image_file == 'None': + self.image_file = None + self.extra_properties = kwargs.get('extra_properties') self.nic_config_pb_loc = kwargs.get('nic_config_pb_loc') @@ -299,10 +304,6 @@ class ImageSettings: raise ImageSettingsError( 'URL or image file must be set or image must already exist') - if self.url and self.image_file: - raise ImageSettingsError( - 'Please set either URL or image file, not both') - if not self.image_user: raise ImageSettingsError('Image user is required') diff --git a/snaps/openstack/create_instance.py b/snaps/openstack/create_instance.py index 6b9a122..2fbeb79 100644 --- a/snaps/openstack/create_instance.py +++ b/snaps/openstack/create_instance.py @@ -738,8 +738,12 @@ class VmInstanceSettings: waiting obtaining an SSH connection to a VM :param availability_zone: the name of the compute server on which to deploy the VM (optional) - :param userdata: the cloud-init script to run after the VM has been - started + :param userdata: the string contents of any optional cloud-init script + to execute after the VM has been activated. + This value may also contain a dict who's key value + must contain the key 'cloud-init_file' which denotes + the location of some file containing the cloud-init + script """ self.name = kwargs.get('name') self.flavor = kwargs.get('flavor') diff --git a/snaps/openstack/create_keypairs.py b/snaps/openstack/create_keypairs.py index cc32da3..d0b69cd 100644 --- a/snaps/openstack/create_keypairs.py +++ b/snaps/openstack/create_keypairs.py @@ -56,8 +56,12 @@ class OpenStackKeypair: logger.info('Creating keypair %s...' % self.keypair_settings.name) - self.__keypair = nova_utils.get_keypair_by_name( - self.__nova, self.keypair_settings.name) + try: + self.__keypair = nova_utils.get_keypair_by_name( + self.__nova, self.keypair_settings.name) + except Exception as e: + logger.warn('Cannot load existing keypair - %s', e) + return if not self.__keypair and not cleanup: if self.keypair_settings.public_filepath and os.path.isfile( @@ -109,13 +113,17 @@ class OpenStackKeypair: if (self.keypair_settings.public_filepath and file_utils.file_exists( self.keypair_settings.public_filepath)): - os.chmod(self.keypair_settings.public_filepath, 0o777) - os.remove(self.keypair_settings.public_filepath) + expanded_path = os.path.expanduser( + self.keypair_settings.public_filepath) + os.chmod(expanded_path, 0o755) + os.remove(expanded_path) if (self.keypair_settings.private_filepath and file_utils.file_exists( self.keypair_settings.private_filepath)): - os.chmod(self.keypair_settings.private_filepath, 0o777) - os.remove(self.keypair_settings.private_filepath) + expanded_path = os.path.expanduser( + self.keypair_settings.private_filepath) + os.chmod(expanded_path, 0o755) + os.remove(expanded_path) def get_keypair(self): """ diff --git a/snaps/openstack/create_project.py b/snaps/openstack/create_project.py index 7bfdad1..38505ad 100644 --- a/snaps/openstack/create_project.py +++ b/snaps/openstack/create_project.py @@ -14,7 +14,7 @@ # limitations under the License. import logging -from keystoneclient.exceptions import NotFound +from keystoneclient.exceptions import NotFound, Conflict from snaps.openstack.utils import keystone_utils, neutron_utils, nova_utils @@ -40,6 +40,7 @@ class OpenStackProject: self.__project = None self.__role = None self.__keystone = None + self.__role_name = self.project_settings.name + '-role' def create(self, cleanup=False): """ @@ -56,6 +57,14 @@ class OpenStackProject: elif not cleanup: self.__project = keystone_utils.create_project( self.__keystone, self.project_settings) + for username in self.project_settings.users: + user = keystone_utils.get_user(self.__keystone, username) + if user: + try: + self.assoc_user(user) + except Conflict as e: + logger.warn('Unable to associate user %s due to %s', + user.name, e) else: logger.info('Did not create image due to cleanup mode') @@ -93,6 +102,12 @@ class OpenStackProject: pass self.__project = None + # Final role check in case init was done from an existing instance + role = keystone_utils.get_role_by_name( + self.__keystone, self.__role_name) + if role: + keystone_utils.delete_role(self.__keystone, role) + def get_project(self): """ Returns the OpenStack project object populated on create() @@ -107,8 +122,11 @@ class OpenStackProject: :return: """ if not self.__role: - self.__role = keystone_utils.create_role( - self.__keystone, self.project_settings.name + '-role') + self.__role = keystone_utils.get_role_by_name( + self.__keystone, self.__role_name) + if not self.__role: + self.__role = keystone_utils.create_role( + self.__keystone, self.__role_name) keystone_utils.grant_user_role_to_project(self.__keystone, self.__role, user, self.__project) @@ -161,6 +179,7 @@ class ProjectSettings: (default = 'Default'). Field is used for v3 clients :param description: the description (optional) + :param users: list of users to associat project to (optional) :param enabled: denotes whether or not the user is enabled (default True) """ @@ -175,6 +194,8 @@ class ProjectSettings: else: self.enabled = True + self.users = kwargs.get('users', list()) + if not self.name: raise ProjectSettingsError( "The attribute name is required for ProjectSettings") diff --git a/snaps/openstack/create_security_group.py b/snaps/openstack/create_security_group.py index 5a0d474..34d5952 100644 --- a/snaps/openstack/create_security_group.py +++ b/snaps/openstack/create_security_group.py @@ -15,7 +15,7 @@ import logging import enum -from neutronclient.common.exceptions import NotFound +from neutronclient.common.exceptions import NotFound, Conflict from snaps.openstack.utils import keystone_utils from snaps.openstack.utils import neutron_utils @@ -78,9 +78,13 @@ class OpenStackSecurityGroup: # Create the custom rules for sec_grp_rule_setting in self.sec_grp_settings.rule_settings: - custom_rule = neutron_utils.create_security_group_rule( - self.__neutron, sec_grp_rule_setting) - self.__rules[sec_grp_rule_setting] = custom_rule + try: + custom_rule = neutron_utils.create_security_group_rule( + self.__neutron, sec_grp_rule_setting) + self.__rules[sec_grp_rule_setting] = custom_rule + except Conflict as e: + logger.warn('Unable to create rule due to conflict - %s', + e) # Refresh security group object to reflect the new rules added self.__security_group = neutron_utils.get_security_group( @@ -236,6 +240,7 @@ class SecurityGroupSettings: if isinstance(rule_setting, SecurityGroupRuleSettings): self.rule_settings.append(rule_setting) else: + rule_setting['sec_grp_name'] = self.name self.rule_settings.append(SecurityGroupRuleSettings( **rule_setting)) diff --git a/snaps/openstack/os_credentials.py b/snaps/openstack/os_credentials.py index bb68215..6f25237 100644 --- a/snaps/openstack/os_credentials.py +++ b/snaps/openstack/os_credentials.py @@ -100,7 +100,7 @@ class OSCreds: self.project_domain_name = kwargs['project_domain_name'] if kwargs.get('interface') is None: - self.interface = 'admin' + self.interface = 'public' else: self.interface = kwargs['interface'] diff --git a/snaps/openstack/tests/conf/os_credentials_tests.py b/snaps/openstack/tests/conf/os_credentials_tests.py index b63a91d..5efb32c 100644 --- a/snaps/openstack/tests/conf/os_credentials_tests.py +++ b/snaps/openstack/tests/conf/os_credentials_tests.py @@ -151,7 +151,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('Default', os_creds.user_domain_name) self.assertEqual('default', os_creds.project_domain_id) self.assertEqual('Default', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertFalse(os_creds.cacert) self.assertIsNone(os_creds.proxy_settings) self.assertIsNone(os_creds.region_name) @@ -172,7 +172,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('Default', os_creds.user_domain_name) self.assertEqual('default', os_creds.project_domain_id) self.assertEqual('Default', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertFalse(os_creds.cacert) self.assertIsNone(os_creds.proxy_settings) self.assertIsNone(os_creds.region_name) @@ -196,7 +196,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('Default', os_creds.user_domain_name) self.assertEqual('default', os_creds.project_domain_id) self.assertEqual('Default', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertTrue(os_creds.cacert) self.assertIsNone(os_creds.proxy_settings) self.assertEqual('test_region', os_creds.region_name) @@ -220,7 +220,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('Default', os_creds.user_domain_name) self.assertEqual('default', os_creds.project_domain_id) self.assertEqual('Default', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertTrue(os_creds.cacert) self.assertIsNone(os_creds.proxy_settings) self.assertEqual('test_region', os_creds.region_name) @@ -242,7 +242,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('Default', os_creds.user_domain_name) self.assertEqual('default', os_creds.project_domain_id) self.assertEqual('Default', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertFalse(os_creds.cacert) self.assertEqual('foo', os_creds.proxy_settings.host) self.assertEqual('1234', os_creds.proxy_settings.port) @@ -270,7 +270,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('domain2', os_creds.user_domain_name) self.assertEqual('domain3', os_creds.project_domain_id) self.assertEqual('domain4', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertFalse(os_creds.cacert) self.assertEqual('foo', os_creds.proxy_settings.host) self.assertEqual('1234', os_creds.proxy_settings.port) @@ -295,7 +295,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('domain2', os_creds.user_domain_name) self.assertEqual('domain3', os_creds.project_domain_id) self.assertEqual('domain4', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertFalse(os_creds.cacert) self.assertEqual('foo', os_creds.proxy_settings.host) self.assertEqual('1234', os_creds.proxy_settings.port) @@ -319,7 +319,7 @@ class OSCredsUnitTests(unittest.TestCase): self.assertEqual('Default', os_creds.user_domain_name) self.assertEqual('default', os_creds.project_domain_id) self.assertEqual('Default', os_creds.project_domain_name) - self.assertEqual('admin', os_creds.interface) + self.assertEqual('public', os_creds.interface) self.assertFalse(os_creds.cacert) self.assertEqual('foo', os_creds.proxy_settings.host) self.assertEqual('1234', os_creds.proxy_settings.port) diff --git a/snaps/openstack/tests/create_image_tests.py b/snaps/openstack/tests/create_image_tests.py index 7a6db86..f70a71c 100644 --- a/snaps/openstack/tests/create_image_tests.py +++ b/snaps/openstack/tests/create_image_tests.py @@ -77,19 +77,6 @@ class ImageSettingsUnitTests(unittest.TestCase): ImageSettings( **{'name': 'foo', 'image_user': 'bar', 'format': 'qcow2'}) - def test_name_user_format_url_file_only(self): - with self.assertRaises(ImageSettingsError): - ImageSettings(name='foo', image_user='bar', img_format='qcow2', - url='http://foo.com', - image_file='/foo/bar.qcow') - - def test_config_with_name_user_format_url_file_only(self): - with self.assertRaises(ImageSettingsError): - ImageSettings( - **{'name': 'foo', 'image_user': 'bar', 'format': 'qcow2', - 'download_url': 'http://foo.com', - 'image_file': '/foo/bar.qcow'}) - def test_name_user_format_url_only(self): settings = ImageSettings(name='foo', image_user='bar', img_format='qcow2', url='http://foo.com') diff --git a/snaps/openstack/tests/create_project_tests.py b/snaps/openstack/tests/create_project_tests.py index 0e1d0ae..aa9dcfb 100644 --- a/snaps/openstack/tests/create_project_tests.py +++ b/snaps/openstack/tests/create_project_tests.py @@ -49,6 +49,7 @@ class ProjectSettingsUnitTests(unittest.TestCase): self.assertEqual('Default', settings.domain_name) self.assertIsNone(settings.description) self.assertTrue(settings.enabled) + self.assertEqual(list(), settings.users) def test_config_with_name_only(self): settings = ProjectSettings(**{'name': 'foo'}) @@ -56,23 +57,29 @@ class ProjectSettingsUnitTests(unittest.TestCase): self.assertEqual('Default', settings.domain_name) self.assertIsNone(settings.description) self.assertTrue(settings.enabled) + self.assertEqual(list(), settings.users) def test_all(self): - settings = ProjectSettings(name='foo', domain='bar', - description='foobar', enabled=False) + users = ['test1', 'test2'] + settings = ProjectSettings( + name='foo', domain='bar', description='foobar', enabled=False, + users=users) self.assertEqual('foo', settings.name) self.assertEqual('bar', settings.domain_name) self.assertEqual('foobar', settings.description) self.assertFalse(settings.enabled) + self.assertEqual(users, settings.users) def test_config_all(self): + users = ['test1', 'test2'] settings = ProjectSettings( **{'name': 'foo', 'domain': 'bar', 'description': 'foobar', - 'enabled': False}) + 'enabled': False, 'users': users}) self.assertEqual('foo', settings.name) self.assertEqual('bar', settings.domain_name) self.assertEqual('foobar', settings.description) self.assertFalse(settings.enabled) + self.assertEqual(users, settings.users) class CreateProjectSuccessTests(OSComponentTestCase): diff --git a/snaps/openstack/utils/glance_utils.py b/snaps/openstack/utils/glance_utils.py index 2606e32..a127ad3 100644 --- a/snaps/openstack/utils/glance_utils.py +++ b/snaps/openstack/utils/glance_utils.py @@ -168,7 +168,7 @@ def __create_image_v2(glance, image_settings): """ cleanup_temp_file = False image_file = None - if image_settings.image_file: + if image_settings.image_file is not None: image_filename = image_settings.image_file elif image_settings.url: file_name = str(uuid.uuid4()) @@ -199,7 +199,7 @@ def __create_image_v2(glance, image_settings): kwargs.update(image_settings.extra_properties) os_image = glance.images.create(**kwargs) - image_file = open(image_filename, 'rb') + image_file = open(os.path.expanduser(image_filename), 'rb') glance.images.upload(os_image['id'], image_file) except: logger.error('Unexpected exception creating image. Rolling back') diff --git a/snaps/openstack/utils/keystone_utils.py b/snaps/openstack/utils/keystone_utils.py index b36c19f..46f6fb8 100644 --- a/snaps/openstack/utils/keystone_utils.py +++ b/snaps/openstack/utils/keystone_utils.py @@ -176,6 +176,7 @@ def create_project(keystone, project_settings): enabled=project_settings.enabled) domain_id = os_project.domain_id + logger.info('Created project with name - %s', project_settings.name) return Project( name=os_project.name, project_id=os_project.id, domain_id=domain_id) @@ -186,6 +187,7 @@ def delete_project(keystone, project): :param keystone: the Keystone clien :param project: the SNAPS-OO Project domain object """ + logger.info('Deleting project with name - %s', project.name) if keystone.version == V2_VERSION_STR: keystone.tenants.delete(project.id) else: @@ -273,6 +275,7 @@ def create_user(keystone, user_settings): project=os_project) if os_user: + logger.info('Created user with name - %s', os_user.name) return User(name=os_user.name, user_id=os_user.id) @@ -282,6 +285,7 @@ def delete_user(keystone, user): :param keystone: the Keystone client :param user: the SNAPS-OO User domain object """ + logger.info('Deleting user with name - %s', user.name) keystone.users.delete(user.id) @@ -337,6 +341,7 @@ def create_role(keystone, name): :return: a SNAPS-OO Role domain object """ role = keystone.roles.create(name) + logger.info('Created role with name - %s', role.name) return Role(name=role.name, role_id=role.id) @@ -347,6 +352,7 @@ def delete_role(keystone, role): :param role: the SNAPS-OO Role domain object to delete :return: """ + logger.info('Deleting role with name - %s', role.name) keystone.roles.delete(role.id) @@ -361,6 +367,7 @@ def grant_user_role_to_project(keystone, role, user, project): """ os_role = get_role_by_id(keystone, role.id) + logger.info('Granting role %s to project %s', role.name, project) if keystone.version == V2_VERSION_STR: keystone.roles.add_user_role(user, os_role, tenant=project) else: diff --git a/snaps/openstack/utils/nova_utils.py b/snaps/openstack/utils/nova_utils.py index fe53211..1665fd0 100644 --- a/snaps/openstack/utils/nova_utils.py +++ b/snaps/openstack/utils/nova_utils.py @@ -22,6 +22,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from novaclient.client import Client from novaclient.exceptions import NotFound +from snaps import file_utils from snaps.domain.flavor import Flavor from snaps.domain.keypair import Keypair from snaps.domain.project import ComputeQuotas @@ -86,6 +87,18 @@ def create_server(nova, neutron, glance, instance_settings, image_settings, image = glance_utils.get_image(glance, image_settings=image_settings) if image: + userdata = None + if instance_settings.userdata: + if isinstance(instance_settings.userdata, str): + userdata = instance_settings.userdata + '\n' + elif (isinstance(instance_settings.userdata, dict) and + 'script_file' in instance_settings.userdata): + try: + userdata = file_utils.read_file( + instance_settings.userdata['script_file']) + except Exception as e: + logger.warn('error reading userdata file %s - %s', + instance_settings.userdata, e) args = {'name': instance_settings.name, 'flavor': flavor, 'image': image, @@ -93,7 +106,7 @@ def create_server(nova, neutron, glance, instance_settings, image_settings, 'key_name': keypair_name, 'security_groups': instance_settings.security_group_names, - 'userdata': instance_settings.userdata} + 'userdata': userdata} if instance_settings.availability_zone: args['availability_zone'] = instance_settings.availability_zone @@ -264,6 +277,58 @@ def public_key_openssh(keys): serialization.PublicFormat.OpenSSH) +def save_keys_to_files(keys=None, pub_file_path=None, priv_file_path=None): + """ + Saves the generated RSA generated keys to the filesystem + :param keys: the keys to save generated by cryptography + :param pub_file_path: the path to the public keys + :param priv_file_path: the path to the private keys + """ + if keys: + if pub_file_path: + # To support '~' + pub_expand_file = os.path.expanduser(pub_file_path) + pub_dir = os.path.dirname(pub_expand_file) + + if not os.path.isdir(pub_dir): + os.mkdir(pub_dir) + + public_handle = None + try: + public_handle = open(pub_expand_file, 'wb') + public_bytes = keys.public_key().public_bytes( + serialization.Encoding.OpenSSH, + serialization.PublicFormat.OpenSSH) + public_handle.write(public_bytes) + finally: + if public_handle: + public_handle.close() + + os.chmod(pub_expand_file, 0o600) + logger.info("Saved public key to - " + pub_expand_file) + if priv_file_path: + # To support '~' + priv_expand_file = os.path.expanduser(priv_file_path) + priv_dir = os.path.dirname(priv_expand_file) + if not os.path.isdir(priv_dir): + os.mkdir(priv_dir) + + private_handle = None + try: + private_handle = open(priv_expand_file, 'wb') + private_handle.write( + keys.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption())) + finally: + if private_handle: + private_handle.close() + + os.chmod(priv_expand_file, 0o600) + logger.info("Saved private key to - " + priv_expand_file) + + def upload_keypair_file(nova, name, file_path): """ Uploads a public key from a file |