Enhanced launch.py app to support all types.

Added application support for users, projects, and
security groups. In addition, added support for multiple
credential sets so one can manage multiple projects
potentially on multiple clouds.

Added Jinja2 template support for substituting values
in the template file with values contained in the
environment file.

Added ansible substituion value for a VM's floating IP.
Changed credentials interface attribute's default value
from 'admin' to 'public'.

Added optional pre_sleep_time attribute to ansible execution
that will wait for a given number of seconds prior to attempting
to apply a playbook to a set of VMs.

JIRA: SNAPS-4, SNAPS-5, SNAPS-6, SNAPS-26

Change-Id: I67b8d69a3b06a43631d80e8fe0c56e02773dbfbe
Signed-off-by: spisarski <s.pisarski@cablelabs.com>

1 files changed, 222 insertions, 191 deletions

diff --git a/examples/launch.py b/examples/launch.py
index 65142ef..f5d3bea 100644
--- a/examples/launch.py
+++ b/examples/launch.py
@@ -20,23 +20,82 @@ import argparse
 import logging
 import re
 
+import time
+from jinja2 import Environment, FileSystemLoader
 import os
+import yaml
+
 from snaps import file_utils
 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
 from snaps.openstack.create_image import ImageSettings, OpenStackImage
 from snaps.openstack.create_instance import VmInstanceSettings
-from snaps.openstack.create_keypairs import KeypairSettings
-from snaps.openstack.create_network import PortSettings, NetworkSettings
-from snaps.openstack.create_router import RouterSettings
+from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
+from snaps.openstack.create_network import (
+    PortSettings, NetworkSettings, OpenStackNetwork)
+from snaps.openstack.create_project import OpenStackProject, ProjectSettings
+from snaps.openstack.create_router import RouterSettings, OpenStackRouter
+from snaps.openstack.create_security_group import (
+    OpenStackSecurityGroup, SecurityGroupSettings)
+from snaps.openstack.create_user import OpenStackUser, UserSettings
 from snaps.openstack.os_credentials import OSCreds, ProxySettings
 from snaps.openstack.utils import deploy_utils
 from snaps.provisioning import ansible_utils
 
 __author__ = 'spisarski'
 
-logger = logging.getLogger('deploy_venv')
+logger = logging.getLogger('snaps_launcher')
 
 ARG_NOT_SET = "argument not set"
+DEFAULT_CREDS_KEY = 'admin'
+
+
+def __get_creds_dict(os_conn_config):
+    """
+    Returns a dict of OSCreds where the key is the creds name.
+    For backwards compatibility, credentials not contained in a list (only
+    one) will be returned with the key of None
+    :param os_conn_config: the credential configuration
+    :return: a dict of OSCreds objects
+    """
+    if 'connection' in os_conn_config:
+        return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
+    elif 'connections' in os_conn_config:
+        out = dict()
+        for os_conn_dict in os_conn_config['connections']:
+            config = os_conn_dict.get('connection')
+            if not config:
+                raise Exception('Invalid connection format')
+
+            name = config.get('name')
+            if not name:
+                raise Exception('Connection config requires a name field')
+
+            out[name] = __get_os_credentials(os_conn_dict)
+        return out
+
+
+def __get_creds(os_creds_dict, os_user_dict, inst_config):
+    """
+    Returns the appropriate credentials
+    :param os_creds_dict: a dictionary of OSCreds objects where the name is the
+                          key
+    :param os_user_dict: a dictionary of OpenStackUser objects where the name
+                         is the key
+    :param inst_config:
+    :return: an OSCreds instance or None
+    """
+    os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
+    if 'os_user' in inst_config:
+        os_user_conf = inst_config['os_user']
+        if 'name' in os_user_conf:
+            user_creator = os_user_dict.get(os_user_conf['name'])
+            if user_creator:
+                return user_creator.get_os_creds(
+                    project_name=os_user_conf.get('project_name'))
+    elif 'os_creds_name' in inst_config:
+        if 'os_creds_name' in inst_config:
+            os_creds = os_creds_dict[inst_config['os_creds_name']]
+    return os_creds
 
 
 def __get_os_credentials(os_conn_config):
@@ -46,17 +105,30 @@ def __get_os_credentials(os_conn_config):
     :param os_conn_config: The configuration holding the credentials
     :return: an OSCreds instance
     """
+    config = os_conn_config.get('connection')
+    if not config:
+        raise Exception('Invalid connection configuration')
+
     proxy_settings = None
-    http_proxy = os_conn_config.get('http_proxy')
+    http_proxy = config.get('http_proxy')
     if http_proxy:
         tokens = re.split(':', http_proxy)
-        ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd')
+        ssh_proxy_cmd = config.get('ssh_proxy_cmd')
         proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
                                        ssh_proxy_cmd=ssh_proxy_cmd)
+    else:
+        if 'proxy_settings' in config:
+            host = config['proxy_settings'].get('host')
+            port = config['proxy_settings'].get('port')
+            if host and host != 'None' and port and port != 'None':
+                proxy_settings = ProxySettings(**config['proxy_settings'])
+
+    if proxy_settings:
+        config['proxy_settings'] = proxy_settings
+    else:
+        del config['proxy_settings']
 
-    os_conn_config['proxy_settings'] = proxy_settings
-
-    return OSCreds(**os_conn_config)
+    return OSCreds(**config)
 
 
 def __parse_ports_config(config):
@@ -71,155 +143,46 @@ def __parse_ports_config(config):
     return out
 
 
-def __create_flavors(os_conn_config, flavors_config, cleanup=False):
-    """
-    Returns a dictionary of flavors where the key is the image name and the
-    value is the image object
-    :param os_conn_config: The OpenStack connection credentials
-    :param flavors_config: The list of image configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    flavors = {}
-
-    if flavors_config:
-        try:
-            for flavor_config_dict in flavors_config:
-                flavor_config = flavor_config_dict.get('flavor')
-                if flavor_config and flavor_config.get('name'):
-                    flavor_creator = OpenStackFlavor(
-                        __get_os_credentials(os_conn_config),
-                        FlavorSettings(**flavor_config))
-                    flavor_creator.create(cleanup=cleanup)
-                    flavors[flavor_config['name']] = flavor_creator
-        except Exception as e:
-            for key, flavor_creator in flavors.items():
-                flavor_creator.clean()
-            raise e
-        logger.info('Created configured flavors')
-
-    return flavors
-
-
-def __create_images(os_conn_config, images_config, cleanup=False):
-    """
-    Returns a dictionary of images where the key is the image name and the
-    value is the image object
-    :param os_conn_config: The OpenStack connection credentials
-    :param images_config: The list of image configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    images = {}
-
-    if images_config:
-        try:
-            for image_config_dict in images_config:
-                image_config = image_config_dict.get('image')
-                if image_config and image_config.get('name'):
-                    images[image_config['name']] = deploy_utils.create_image(
-                        __get_os_credentials(os_conn_config),
-                        ImageSettings(**image_config), cleanup)
-        except Exception as e:
-            for key, image_creator in images.items():
-                image_creator.clean()
-            raise e
-        logger.info('Created configured images')
-
-    return images
-
-
-def __create_networks(os_conn_config, network_confs, cleanup=False):
-    """
-    Returns a dictionary of networks where the key is the network name and the
-    value is the network object
-    :param os_conn_config: The OpenStack connection credentials
-    :param network_confs: The list of network configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    network_dict = {}
-
-    if network_confs:
-        try:
-            for network_conf in network_confs:
-                net_name = network_conf['network']['name']
-                os_creds = __get_os_credentials(os_conn_config)
-                network_dict[net_name] = deploy_utils.create_network(
-                    os_creds, NetworkSettings(**network_conf['network']),
-                    cleanup)
-        except Exception as e:
-            for key, net_creator in network_dict.items():
-                net_creator.clean()
-            raise e
-
-        logger.info('Created configured networks')
-
-    return network_dict
-
-
-def __create_routers(os_conn_config, router_confs, cleanup=False):
+def __create_instances(os_creds_dict, creator_class, config_class, config,
+                       config_key, cleanup=False, os_users_dict=None):
     """
-    Returns a dictionary of networks where the key is the network name and the
-    value is the network object
-    :param os_conn_config: The OpenStack connection credentials
-    :param router_confs: The list of router configurations
+    Returns a dictionary of SNAPS creator objects where the key is the name
+    :param os_creds_dict: Dictionary of OSCreds objects where the key is the
+                          name
+    :param config: The list of configurations for the same type
+    :param config_key: The list of configurations for the same type
     :param cleanup: Denotes whether or not this is being called for cleanup
     :return: dictionary
     """
-    router_dict = {}
-    os_creds = __get_os_credentials(os_conn_config)
+    out = {}
 
-    if router_confs:
-        try:
-            for router_conf in router_confs:
-                router_name = router_conf['router']['name']
-                router_dict[router_name] = deploy_utils.create_router(
-                    os_creds, RouterSettings(**router_conf['router']), cleanup)
-        except Exception as e:
-            for key, router_creator in router_dict.items():
-                router_creator.clean()
-            raise e
-
-        logger.info('Created configured networks')
-
-    return router_dict
-
-
-def __create_keypairs(os_conn_config, keypair_confs, cleanup=False):
-    """
-    Returns a dictionary of keypairs where the key is the keypair name and the
-    value is the keypair object
-    :param os_conn_config: The OpenStack connection credentials
-    :param keypair_confs: The list of keypair configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    keypairs_dict = {}
-    if keypair_confs:
+    if config:
         try:
-            for keypair_dict in keypair_confs:
-                keypair_config = keypair_dict['keypair']
-                kp_settings = KeypairSettings(**keypair_config)
-                keypairs_dict[
-                    keypair_config['name']] = deploy_utils.create_keypair(
-                    __get_os_credentials(os_conn_config), kp_settings, cleanup)
+            for config_dict in config:
+                inst_config = config_dict.get(config_key)
+                if inst_config:
+                    creator = creator_class(
+                        __get_creds(os_creds_dict, os_users_dict, inst_config),
+                        config_class(**inst_config))
+                    creator.create(cleanup=cleanup)
+                    out[inst_config['name']] = creator
+            logger.info('Created configured %s', config_key)
         except Exception as e:
-            for key, keypair_creator in keypairs_dict.items():
-                keypair_creator.clean()
-            raise e
-
-        logger.info('Created configured keypairs')
+            logger.error('Unexpected error instantiating creator [%s] '
+                         'with exception %s', creator_class, e)
 
-    return keypairs_dict
+    return out
 
 
-def __create_instances(os_conn_config, instances_config, image_dict,
-                       keypairs_dict, cleanup=False):
+def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
+                          image_dict, keypairs_dict, cleanup=False):
     """
-    Returns a dictionary of instances where the key is the instance name and
-    the value is the VM object
-    :param os_conn_config: The OpenStack connection credentials
+    Returns a dictionary of OpenStackVmInstance objects where the key is the
+    instance name
+    :param os_creds_dict: Dictionary of OSCreds objects where the key is the
+                          name
+    :param os_users_dict: Dictionary of OpenStackUser objects where the key is
+                          the username
     :param instances_config: The list of VM instance configurations
     :param image_dict: A dictionary of images that will probably be used to
                        instantiate the VM instance
@@ -228,8 +191,6 @@ def __create_instances(os_conn_config, instances_config, image_dict,
     :param cleanup: Denotes whether or not this is being called for cleanup
     :return: dictionary
     """
-    os_creds = __get_os_credentials(os_conn_config)
-
     vm_dict = {}
 
     if instances_config:
@@ -245,7 +206,9 @@ def __create_instances(os_conn_config, instances_config, image_dict,
                             kp_name = conf.get('keypair_name')
                             vm_dict[conf[
                                 'name']] = deploy_utils.create_vm_instance(
-                                os_creds, instance_settings,
+                                __get_creds(
+                                    os_creds_dict, os_users_dict, conf),
+                                instance_settings,
                                 image_creator.image_settings,
                                 keypair_creator=keypairs_dict[kp_name],
                                 cleanup=cleanup)
@@ -258,24 +221,19 @@ def __create_instances(os_conn_config, instances_config, image_dict,
                 else:
                     raise Exception('Instance configuration is None. Cannot '
                                     'instantiate')
+            logger.info('Created configured instances')
         except Exception as e:
-            logger.error('Unexpected error creating instances. Attempting to '
-                         'cleanup environment - %s', e)
-            for key, inst_creator in vm_dict.items():
-                inst_creator.clean()
-            raise e
-
-        logger.info('Created configured instances')
+            logger.error('Unexpected error creating VM instances - %s', e)
     return vm_dict
 
 
-def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict,
+def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
                               image_dict, flavor_dict, env_file):
     """
     Applies ansible playbooks to running VMs with floating IPs
     :param ansible_configs: a list of Ansible configurations
-    :param os_conn_config: the OpenStack connection configuration used to
-                           create an OSCreds instance
+    :param os_creds_dict: Dictionary of OSCreds objects where the key is the
+                          name
     :param vm_dict: the dictionary of newly instantiated VMs where the name is
                     the key
     :param image_dict: the dictionary of newly instantiated images where the
@@ -303,7 +261,16 @@ def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict,
 
         # Apply playbooks
         for ansible_config in ansible_configs:
-            os_creds = __get_os_credentials(os_conn_config)
+            if 'pre_sleep_time' in ansible_config:
+                try:
+                    sleep_time = int(ansible_config['pre_sleep_time'])
+                    logger.info('Waiting %s seconds to apply playbooks',
+                                sleep_time)
+                    time.sleep(sleep_time)
+                except:
+                    pass
+
+            os_creds = os_creds_dict.get(None, 'admin')
             __apply_ansible_playbook(ansible_config, os_creds, vm_dict,
                                      image_dict, flavor_dict)
 
@@ -341,8 +308,8 @@ def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
             if retval != 0:
                 # Not a fatal type of event
                 logger.warning(
-                    'Unable to apply playbook found at location - ' +
-                    ansible_config('playbook_location'))
+                    'Unable to apply playbook found at location - %s',
+                    ansible_config.get('playbook_location'))
 
 
 def __get_connection_info(ansible_config, vm_dict):
@@ -442,6 +409,8 @@ def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
         return __get_os_creds_variable_value(var_config_values, os_creds)
     if var_config_values['type'] == 'port':
         return __get_vm_port_variable_value(var_config_values, vm_dict)
+    if var_config_values['type'] == 'floating_ip':
+        return __get_vm_fip_variable_value(var_config_values, vm_dict)
     if var_config_values['type'] == 'image':
         return __get_image_variable_value(var_config_values, image_dict)
     if var_config_values['type'] == 'flavor':
@@ -522,6 +491,25 @@ def __get_vm_port_variable_value(var_config_values, vm_dict):
                     return vm.get_port_ip(port_name)
 
 
+def __get_vm_fip_variable_value(var_config_values, vm_dict):
+    """
+    Returns the floating IP value if found
+    :param var_config_values: the configuration dictionary
+    :param vm_dict: the dictionary containing all VMs where the key is the VM's
+                    name
+    :return: the floating IP string value or None
+    """
+    fip_name = var_config_values.get('fip_name')
+    vm_name = var_config_values.get('vm_name')
+
+    if vm_name:
+        vm = vm_dict.get(vm_name)
+        if vm:
+            fip = vm.get_floating_ip(fip_name)
+            if fip:
+                return fip.ip
+
+
 def __get_image_variable_value(var_config_values, image_dict):
     """
     Returns the associated image value
@@ -585,53 +573,93 @@ def main(arguments):
     logging.basicConfig(level=log_level)
 
     logger.info('Starting to Deploy')
-    config = file_utils.read_yaml(arguments.environment)
-    logger.debug('Read configuration file - ' + arguments.environment)
+
+    # Apply env_file/substitution file to template
+    env = Environment(loader=FileSystemLoader(
+        searchpath=os.path.dirname(arguments.tmplt_file)))
+    template = env.get_template(os.path.basename(arguments.tmplt_file))
+
+    env_dict = dict()
+    if arguments.env_file:
+        env_dict = file_utils.read_yaml(arguments.env_file)
+    output = template.render(**env_dict)
+
+    config = yaml.load(output)
 
     if config:
         os_config = config.get('openstack')
 
-        os_conn_config = None
         creators = list()
         vm_dict = dict()
         images_dict = dict()
         flavors_dict = dict()
+        os_creds_dict = dict()
+        clean = arguments.clean is not ARG_NOT_SET
 
         if os_config:
+            os_creds_dict = __get_creds_dict(os_config)
+
             try:
-                os_conn_config = os_config.get('connection')
+                # Create projects
+                projects_dict = __create_instances(
+                    os_creds_dict, OpenStackProject, ProjectSettings,
+                    os_config.get('projects'), 'project', clean)
+                creators.append(projects_dict)
+
+                # Create users
+                users_dict = __create_instances(
+                    os_creds_dict, OpenStackUser, UserSettings,
+                    os_config.get('users'), 'user', clean)
+                creators.append(users_dict)
+
+                # Associate new users to projects
+                if not clean:
+                    for project_creator in projects_dict.values():
+                        users = project_creator.project_settings.users
+                        for user_name in users:
+                            user_creator = users_dict.get(user_name)
+                            if user_creator:
+                                project_creator.assoc_user(
+                                    user_creator.get_user())
 
                 # Create flavors
-                flavors_dict = __create_flavors(
-                    os_conn_config, os_config.get('flavors'),
-                    arguments.clean is not ARG_NOT_SET)
+                flavors_dict = __create_instances(
+                    os_creds_dict, OpenStackFlavor, FlavorSettings,
+                    os_config.get('flavors'), 'flavor', clean, users_dict)
                 creators.append(flavors_dict)
 
                 # Create images
-                images_dict = __create_images(
-                    os_conn_config, os_config.get('images'),
-                    arguments.clean is not ARG_NOT_SET)
+                images_dict = __create_instances(
+                    os_creds_dict, OpenStackImage, ImageSettings,
+                    os_config.get('images'), 'image', clean, users_dict)
                 creators.append(images_dict)
 
-                # Create network
-                creators.append(__create_networks(
-                    os_conn_config, os_config.get('networks'),
-                    arguments.clean is not ARG_NOT_SET))
+                # Create networks
+                creators.append(__create_instances(
+                    os_creds_dict, OpenStackNetwork, NetworkSettings,
+                    os_config.get('networks'), 'network', clean, users_dict))
 
                 # Create routers
-                creators.append(__create_routers(
-                    os_conn_config, os_config.get('routers'),
-                    arguments.clean is not ARG_NOT_SET))
+                creators.append(__create_instances(
+                    os_creds_dict, OpenStackRouter, RouterSettings,
+                    os_config.get('routers'), 'router', clean, users_dict))
 
                 # Create keypairs
-                keypairs_dict = __create_keypairs(
-                    os_conn_config, os_config.get('keypairs'),
-                    arguments.clean is not ARG_NOT_SET)
+                keypairs_dict = __create_instances(
+                    os_creds_dict, OpenStackKeypair, KeypairSettings,
+                    os_config.get('keypairs'), 'keypair', clean, users_dict)
                 creators.append(keypairs_dict)
 
+                # Create security groups
+                creators.append(__create_instances(
+                    os_creds_dict, OpenStackSecurityGroup,
+                    SecurityGroupSettings,
+                    os_config.get('security_groups'), 'security_group', clean,
+                    users_dict))
+
                 # Create instance
-                vm_dict = __create_instances(
-                    os_conn_config, os_config.get('instances'),
+                vm_dict = __create_vm_instances(
+                    os_creds_dict, users_dict, os_config.get('instances'),
                     images_dict, keypairs_dict,
                     arguments.clean is not ARG_NOT_SET)
                 creators.append(vm_dict)
@@ -641,7 +669,7 @@ def main(arguments):
                 logger.error(
                     'Unexpected error deploying environment. Rolling back due'
                     ' to - ' + str(e))
-                __cleanup(creators)
+                # __cleanup(creators)
                 raise
 
         # Must enter either block
@@ -658,13 +686,13 @@ def main(arguments):
             ansible_config = config.get('ansible')
             if ansible_config and vm_dict:
                 if not __apply_ansible_playbooks(ansible_config,
-                                                 os_conn_config, vm_dict,
+                                                 os_creds_dict, vm_dict,
                                                  images_dict, flavors_dict,
-                                                 arguments.environment):
+                                                 arguments.tmplt_file):
                     logger.error("Problem applying ansible playbooks")
     else:
         logger.error(
-            'Unable to read configuration file - ' + arguments.environment)
+            'Unable to read configuration file - ' + arguments.tmplt_file)
         exit(1)
 
     exit(0)
@@ -698,8 +726,11 @@ if __name__ == '__main__':
         default=ARG_NOT_SET,
         help='When cleaning, if this is set, the image will be cleaned too')
     parser.add_argument(
-        '-e', '--env', dest='environment', required=True,
-        help='The environment configuration YAML file - REQUIRED')
+        '-t', '--tmplt', dest='tmplt_file', required=True,
+        help='The SNAPS deployment template YAML file - REQUIRED')
+    parser.add_argument(
+        '-e', '--env-file', dest='env_file',
+        help='Yaml file containing substitution values to the env file')
     parser.add_argument(
         '-l', '--log-level', dest='log_level', default='INFO',
         help='Logging Level (INFO|DEBUG)')