Enhanced launch.py app to support all types.

Added application support for users, projects, and
security groups. In addition, added support for multiple
credential sets so one can manage multiple projects
potentially on multiple clouds.

Added Jinja2 template support for substituting values
in the template file with values contained in the
environment file.

Added ansible substituion value for a VM's floating IP.
Changed credentials interface attribute's default value
from 'admin' to 'public'.

Added optional pre_sleep_time attribute to ansible execution
that will wait for a given number of seconds prior to attempting
to apply a playbook to a set of VMs.

JIRA: SNAPS-4, SNAPS-5, SNAPS-6, SNAPS-26

Change-Id: I67b8d69a3b06a43631d80e8fe0c56e02773dbfbe
Signed-off-by: spisarski <s.pisarski@cablelabs.com>

15 files changed, 382 insertions, 245 deletions

diff --git a/docs/how-to-use/VirtEnvDeploy.rst b/docs/how-to-use/VirtEnvDeploy.rst
index f8a1e3c..dd95202 100644
--- a/docs/how-to-use/VirtEnvDeploy.rst
+++ b/docs/how-to-use/VirtEnvDeploy.rst
@@ -29,13 +29,13 @@ Use launcher.py to deploy and clean up example environments.  These examples are
 
     ::
 
-      python launch.py -e ./complex-network/deploy-complex-network.yaml -d
+      python launch.py -t ./complex-network/deploy-complex-network.yaml -d
 
 #. Clean the deployment.
 
     ::
 
-      python launch.py -e ./complex-network/deploy-complex-network.yaml -c
+      python launch.py -t ./complex-network/deploy-complex-network.yaml -c
 
 #. Customize the deployment by changing the yaml file.
 
diff --git a/examples/launch.py b/examples/launch.py
index 65142ef..f5d3bea 100644
--- a/examples/launch.py
+++ b/examples/launch.py
@@ -20,23 +20,82 @@ import argparse
 import logging
 import re
 
+import time
+from jinja2 import Environment, FileSystemLoader
 import os
+import yaml
+
 from snaps import file_utils
 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
 from snaps.openstack.create_image import ImageSettings, OpenStackImage
 from snaps.openstack.create_instance import VmInstanceSettings
-from snaps.openstack.create_keypairs import KeypairSettings
-from snaps.openstack.create_network import PortSettings, NetworkSettings
-from snaps.openstack.create_router import RouterSettings
+from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
+from snaps.openstack.create_network import (
+    PortSettings, NetworkSettings, OpenStackNetwork)
+from snaps.openstack.create_project import OpenStackProject, ProjectSettings
+from snaps.openstack.create_router import RouterSettings, OpenStackRouter
+from snaps.openstack.create_security_group import (
+    OpenStackSecurityGroup, SecurityGroupSettings)
+from snaps.openstack.create_user import OpenStackUser, UserSettings
 from snaps.openstack.os_credentials import OSCreds, ProxySettings
 from snaps.openstack.utils import deploy_utils
 from snaps.provisioning import ansible_utils
 
 __author__ = 'spisarski'
 
-logger = logging.getLogger('deploy_venv')
+logger = logging.getLogger('snaps_launcher')
 
 ARG_NOT_SET = "argument not set"
+DEFAULT_CREDS_KEY = 'admin'
+
+
+def __get_creds_dict(os_conn_config):
+    """
+    Returns a dict of OSCreds where the key is the creds name.
+    For backwards compatibility, credentials not contained in a list (only
+    one) will be returned with the key of None
+    :param os_conn_config: the credential configuration
+    :return: a dict of OSCreds objects
+    """
+    if 'connection' in os_conn_config:
+        return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
+    elif 'connections' in os_conn_config:
+        out = dict()
+        for os_conn_dict in os_conn_config['connections']:
+            config = os_conn_dict.get('connection')
+            if not config:
+                raise Exception('Invalid connection format')
+
+            name = config.get('name')
+            if not name:
+                raise Exception('Connection config requires a name field')
+
+            out[name] = __get_os_credentials(os_conn_dict)
+        return out
+
+
+def __get_creds(os_creds_dict, os_user_dict, inst_config):
+    """
+    Returns the appropriate credentials
+    :param os_creds_dict: a dictionary of OSCreds objects where the name is the
+                          key
+    :param os_user_dict: a dictionary of OpenStackUser objects where the name
+                         is the key
+    :param inst_config:
+    :return: an OSCreds instance or None
+    """
+    os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
+    if 'os_user' in inst_config:
+        os_user_conf = inst_config['os_user']
+        if 'name' in os_user_conf:
+            user_creator = os_user_dict.get(os_user_conf['name'])
+            if user_creator:
+                return user_creator.get_os_creds(
+                    project_name=os_user_conf.get('project_name'))
+    elif 'os_creds_name' in inst_config:
+        if 'os_creds_name' in inst_config:
+            os_creds = os_creds_dict[inst_config['os_creds_name']]
+    return os_creds
 
 
 def __get_os_credentials(os_conn_config):
@@ -46,17 +105,30 @@ def __get_os_credentials(os_conn_config):
     :param os_conn_config: The configuration holding the credentials
     :return: an OSCreds instance
     """
+    config = os_conn_config.get('connection')
+    if not config:
+        raise Exception('Invalid connection configuration')
+
     proxy_settings = None
-    http_proxy = os_conn_config.get('http_proxy')
+    http_proxy = config.get('http_proxy')
     if http_proxy:
         tokens = re.split(':', http_proxy)
-        ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd')
+        ssh_proxy_cmd = config.get('ssh_proxy_cmd')
         proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
                                        ssh_proxy_cmd=ssh_proxy_cmd)
+    else:
+        if 'proxy_settings' in config:
+            host = config['proxy_settings'].get('host')
+            port = config['proxy_settings'].get('port')
+            if host and host != 'None' and port and port != 'None':
+                proxy_settings = ProxySettings(**config['proxy_settings'])
+
+    if proxy_settings:
+        config['proxy_settings'] = proxy_settings
+    else:
+        del config['proxy_settings']
 
-    os_conn_config['proxy_settings'] = proxy_settings
-
-    return OSCreds(**os_conn_config)
+    return OSCreds(**config)
 
 
 def __parse_ports_config(config):
@@ -71,155 +143,46 @@ def __parse_ports_config(config):
     return out
 
 
-def __create_flavors(os_conn_config, flavors_config, cleanup=False):
-    """
-    Returns a dictionary of flavors where the key is the image name and the
-    value is the image object
-    :param os_conn_config: The OpenStack connection credentials
-    :param flavors_config: The list of image configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    flavors = {}
-
-    if flavors_config:
-        try:
-            for flavor_config_dict in flavors_config:
-                flavor_config = flavor_config_dict.get('flavor')
-                if flavor_config and flavor_config.get('name'):
-                    flavor_creator = OpenStackFlavor(
-                        __get_os_credentials(os_conn_config),
-                        FlavorSettings(**flavor_config))
-                    flavor_creator.create(cleanup=cleanup)
-                    flavors[flavor_config['name']] = flavor_creator
-        except Exception as e:
-            for key, flavor_creator in flavors.items():
-                flavor_creator.clean()
-            raise e
-        logger.info('Created configured flavors')
-
-    return flavors
-
-
-def __create_images(os_conn_config, images_config, cleanup=False):
-    """
-    Returns a dictionary of images where the key is the image name and the
-    value is the image object
-    :param os_conn_config: The OpenStack connection credentials
-    :param images_config: The list of image configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    images = {}
-
-    if images_config:
-        try:
-            for image_config_dict in images_config:
-                image_config = image_config_dict.get('image')
-                if image_config and image_config.get('name'):
-                    images[image_config['name']] = deploy_utils.create_image(
-                        __get_os_credentials(os_conn_config),
-                        ImageSettings(**image_config), cleanup)
-        except Exception as e:
-            for key, image_creator in images.items():
-                image_creator.clean()
-            raise e
-        logger.info('Created configured images')
-
-    return images
-
-
-def __create_networks(os_conn_config, network_confs, cleanup=False):
-    """
-    Returns a dictionary of networks where the key is the network name and the
-    value is the network object
-    :param os_conn_config: The OpenStack connection credentials
-    :param network_confs: The list of network configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    network_dict = {}
-
-    if network_confs:
-        try:
-            for network_conf in network_confs:
-                net_name = network_conf['network']['name']
-                os_creds = __get_os_credentials(os_conn_config)
-                network_dict[net_name] = deploy_utils.create_network(
-                    os_creds, NetworkSettings(**network_conf['network']),
-                    cleanup)
-        except Exception as e:
-            for key, net_creator in network_dict.items():
-                net_creator.clean()
-            raise e
-
-        logger.info('Created configured networks')
-
-    return network_dict
-
-
-def __create_routers(os_conn_config, router_confs, cleanup=False):
+def __create_instances(os_creds_dict, creator_class, config_class, config,
+                       config_key, cleanup=False, os_users_dict=None):
     """
-    Returns a dictionary of networks where the key is the network name and the
-    value is the network object
-    :param os_conn_config: The OpenStack connection credentials
-    :param router_confs: The list of router configurations
+    Returns a dictionary of SNAPS creator objects where the key is the name
+    :param os_creds_dict: Dictionary of OSCreds objects where the key is the
+                          name
+    :param config: The list of configurations for the same type
+    :param config_key: The list of configurations for the same type
     :param cleanup: Denotes whether or not this is being called for cleanup
     :return: dictionary
     """
-    router_dict = {}
-    os_creds = __get_os_credentials(os_conn_config)
+    out = {}
 
-    if router_confs:
-        try:
-            for router_conf in router_confs:
-                router_name = router_conf['router']['name']
-                router_dict[router_name] = deploy_utils.create_router(
-                    os_creds, RouterSettings(**router_conf['router']), cleanup)
-        except Exception as e:
-            for key, router_creator in router_dict.items():
-                router_creator.clean()
-            raise e
-
-        logger.info('Created configured networks')
-
-    return router_dict
-
-
-def __create_keypairs(os_conn_config, keypair_confs, cleanup=False):
-    """
-    Returns a dictionary of keypairs where the key is the keypair name and the
-    value is the keypair object
-    :param os_conn_config: The OpenStack connection credentials
-    :param keypair_confs: The list of keypair configurations
-    :param cleanup: Denotes whether or not this is being called for cleanup
-    :return: dictionary
-    """
-    keypairs_dict = {}
-    if keypair_confs:
+    if config:
         try:
-            for keypair_dict in keypair_confs:
-                keypair_config = keypair_dict['keypair']
-                kp_settings = KeypairSettings(**keypair_config)
-                keypairs_dict[
-                    keypair_config['name']] = deploy_utils.create_keypair(
-                    __get_os_credentials(os_conn_config), kp_settings, cleanup)
+            for config_dict in config:
+                inst_config = config_dict.get(config_key)
+                if inst_config:
+                    creator = creator_class(
+                        __get_creds(os_creds_dict, os_users_dict, inst_config),
+                        config_class(**inst_config))
+                    creator.create(cleanup=cleanup)
+                    out[inst_config['name']] = creator
+            logger.info('Created configured %s', config_key)
         except Exception as e:
-            for key, keypair_creator in keypairs_dict.items():
-                keypair_creator.clean()
-            raise e
-
-        logger.info('Created configured keypairs')
+            logger.error('Unexpected error instantiating creator [%s] '
+                         'with exception %s', creator_class, e)
 
-    return keypairs_dict
+    return out
 
 
-def __create_instances(os_conn_config, instances_config, image_dict,
-                       keypairs_dict, cleanup=False):
+def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
+                          image_dict, keypairs_dict, cleanup=False):
     """
-    Returns a dictionary of instances where the key is the instance name and
-    the value is the VM object
-    :param os_conn_config: The OpenStack connection credentials
+    Returns a dictionary of OpenStackVmInstance objects where the key is the
+    instance name
+    :param os_creds_dict: Dictionary of OSCreds objects where the key is the
+                          name
+    :param os_users_dict: Dictionary of OpenStackUser objects where the key is
+                          the username
     :param instances_config: The list of VM instance configurations
     :param image_dict: A dictionary of images that will probably be used to
                        instantiate the VM instance
@@ -228,8 +191,6 @@ def __create_instances(os_conn_config, instances_config, image_dict,
     :param cleanup: Denotes whether or not this is being called for cleanup
     :return: dictionary
     """
-    os_creds = __get_os_credentials(os_conn_config)
-
     vm_dict = {}
 
     if instances_config:
@@ -245,7 +206,9 @@ def __create_instances(os_conn_config, instances_config, image_dict,
                             kp_name = conf.get('keypair_name')
                             vm_dict[conf[
                                 'name']] = deploy_utils.create_vm_instance(
-                                os_creds, instance_settings,
+                                __get_creds(
+                                    os_creds_dict, os_users_dict, conf),
+                                instance_settings,
                                 image_creator.image_settings,
                                 keypair_creator=keypairs_dict[kp_name],
                                 cleanup=cleanup)
@@ -258,24 +221,19 @@ def __create_instances(os_conn_config, instances_config, image_dict,
                 else:
                     raise Exception('Instance configuration is None. Cannot '
                                     'instantiate')
+            logger.info('Created configured instances')
         except Exception as e:
-            logger.error('Unexpected error creating instances. Attempting to '
-                         'cleanup environment - %s', e)
-            for key, inst_creator in vm_dict.items():
-                inst_creator.clean()
-            raise e
-
-        logger.info('Created configured instances')
+            logger.error('Unexpected error creating VM instances - %s', e)
     return vm_dict
 
 
-def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict,
+def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
                               image_dict, flavor_dict, env_file):
     """
     Applies ansible playbooks to running VMs with floating IPs
     :param ansible_configs: a list of Ansible configurations
-    :param os_conn_config: the OpenStack connection configuration used to
-                           create an OSCreds instance
+    :param os_creds_dict: Dictionary of OSCreds objects where the key is the
+                          name
     :param vm_dict: the dictionary of newly instantiated VMs where the name is
                     the key
     :param image_dict: the dictionary of newly instantiated images where the
@@ -303,7 +261,16 @@ def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict,
 
         # Apply playbooks
         for ansible_config in ansible_configs:
-            os_creds = __get_os_credentials(os_conn_config)
+            if 'pre_sleep_time' in ansible_config:
+                try:
+                    sleep_time = int(ansible_config['pre_sleep_time'])
+                    logger.info('Waiting %s seconds to apply playbooks',
+                                sleep_time)
+                    time.sleep(sleep_time)
+                except:
+                    pass
+
+            os_creds = os_creds_dict.get(None, 'admin')
             __apply_ansible_playbook(ansible_config, os_creds, vm_dict,
                                      image_dict, flavor_dict)
 
@@ -341,8 +308,8 @@ def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
             if retval != 0:
                 # Not a fatal type of event
                 logger.warning(
-                    'Unable to apply playbook found at location - ' +
-                    ansible_config('playbook_location'))
+                    'Unable to apply playbook found at location - %s',
+                    ansible_config.get('playbook_location'))
 
 
 def __get_connection_info(ansible_config, vm_dict):
@@ -442,6 +409,8 @@ def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
         return __get_os_creds_variable_value(var_config_values, os_creds)
     if var_config_values['type'] == 'port':
         return __get_vm_port_variable_value(var_config_values, vm_dict)
+    if var_config_values['type'] == 'floating_ip':
+        return __get_vm_fip_variable_value(var_config_values, vm_dict)
     if var_config_values['type'] == 'image':
         return __get_image_variable_value(var_config_values, image_dict)
     if var_config_values['type'] == 'flavor':
@@ -522,6 +491,25 @@ def __get_vm_port_variable_value(var_config_values, vm_dict):
                     return vm.get_port_ip(port_name)
 
 
+def __get_vm_fip_variable_value(var_config_values, vm_dict):
+    """
+    Returns the floating IP value if found
+    :param var_config_values: the configuration dictionary
+    :param vm_dict: the dictionary containing all VMs where the key is the VM's
+                    name
+    :return: the floating IP string value or None
+    """
+    fip_name = var_config_values.get('fip_name')
+    vm_name = var_config_values.get('vm_name')
+
+    if vm_name:
+        vm = vm_dict.get(vm_name)
+        if vm:
+            fip = vm.get_floating_ip(fip_name)
+            if fip:
+                return fip.ip
+
+
 def __get_image_variable_value(var_config_values, image_dict):
     """
     Returns the associated image value
@@ -585,53 +573,93 @@ def main(arguments):
     logging.basicConfig(level=log_level)
 
     logger.info('Starting to Deploy')
-    config = file_utils.read_yaml(arguments.environment)
-    logger.debug('Read configuration file - ' + arguments.environment)
+
+    # Apply env_file/substitution file to template
+    env = Environment(loader=FileSystemLoader(
+        searchpath=os.path.dirname(arguments.tmplt_file)))
+    template = env.get_template(os.path.basename(arguments.tmplt_file))
+
+    env_dict = dict()
+    if arguments.env_file:
+        env_dict = file_utils.read_yaml(arguments.env_file)
+    output = template.render(**env_dict)
+
+    config = yaml.load(output)
 
     if config:
         os_config = config.get('openstack')
 
-        os_conn_config = None
         creators = list()
         vm_dict = dict()
         images_dict = dict()
         flavors_dict = dict()
+        os_creds_dict = dict()
+        clean = arguments.clean is not ARG_NOT_SET
 
         if os_config:
+            os_creds_dict = __get_creds_dict(os_config)
+
             try:
-                os_conn_config = os_config.get('connection')
+                # Create projects
+                projects_dict = __create_instances(
+                    os_creds_dict, OpenStackProject, ProjectSettings,
+                    os_config.get('projects'), 'project', clean)
+                creators.append(projects_dict)
+
+                # Create users
+                users_dict = __create_instances(
+                    os_creds_dict, OpenStackUser, UserSettings,
+                    os_config.get('users'), 'user', clean)
+                creators.append(users_dict)
+
+                # Associate new users to projects
+                if not clean:
+                    for project_creator in projects_dict.values():
+                        users = project_creator.project_settings.users
+                        for user_name in users:
+                            user_creator = users_dict.get(user_name)
+                            if user_creator:
+                                project_creator.assoc_user(
+                                    user_creator.get_user())
 
                 # Create flavors
-                flavors_dict = __create_flavors(
-                    os_conn_config, os_config.get('flavors'),
-                    arguments.clean is not ARG_NOT_SET)
+                flavors_dict = __create_instances(
+                    os_creds_dict, OpenStackFlavor, FlavorSettings,
+                    os_config.get('flavors'), 'flavor', clean, users_dict)
                 creators.append(flavors_dict)
 
                 # Create images
-                images_dict = __create_images(
-                    os_conn_config, os_config.get('images'),
-                    arguments.clean is not ARG_NOT_SET)
+                images_dict = __create_instances(
+                    os_creds_dict, OpenStackImage, ImageSettings,
+                    os_config.get('images'), 'image', clean, users_dict)
                 creators.append(images_dict)
 
-                # Create network
-                creators.append(__create_networks(
-                    os_conn_config, os_config.get('networks'),
-                    arguments.clean is not ARG_NOT_SET))
+                # Create networks
+                creators.append(__create_instances(
+                    os_creds_dict, OpenStackNetwork, NetworkSettings,
+                    os_config.get('networks'), 'network', clean, users_dict))
 
                 # Create routers
-                creators.append(__create_routers(
-                    os_conn_config, os_config.get('routers'),
-                    arguments.clean is not ARG_NOT_SET))
+                creators.append(__create_instances(
+                    os_creds_dict, OpenStackRouter, RouterSettings,
+                    os_config.get('routers'), 'router', clean, users_dict))
 
                 # Create keypairs
-                keypairs_dict = __create_keypairs(
-                    os_conn_config, os_config.get('keypairs'),
-                    arguments.clean is not ARG_NOT_SET)
+                keypairs_dict = __create_instances(
+                    os_creds_dict, OpenStackKeypair, KeypairSettings,
+                    os_config.get('keypairs'), 'keypair', clean, users_dict)
                 creators.append(keypairs_dict)
 
+                # Create security groups
+                creators.append(__create_instances(
+                    os_creds_dict, OpenStackSecurityGroup,
+                    SecurityGroupSettings,
+                    os_config.get('security_groups'), 'security_group', clean,
+                    users_dict))
+
                 # Create instance
-                vm_dict = __create_instances(
-                    os_conn_config, os_config.get('instances'),
+                vm_dict = __create_vm_instances(
+                    os_creds_dict, users_dict, os_config.get('instances'),
                     images_dict, keypairs_dict,
                     arguments.clean is not ARG_NOT_SET)
                 creators.append(vm_dict)
@@ -641,7 +669,7 @@ def main(arguments):
                 logger.error(
                     'Unexpected error deploying environment. Rolling back due'
                     ' to - ' + str(e))
-                __cleanup(creators)
+                # __cleanup(creators)
                 raise
 
         # Must enter either block
@@ -658,13 +686,13 @@ def main(arguments):
             ansible_config = config.get('ansible')
             if ansible_config and vm_dict:
                 if not __apply_ansible_playbooks(ansible_config,
-                                                 os_conn_config, vm_dict,
+                                                 os_creds_dict, vm_dict,
                                                  images_dict, flavors_dict,
-                                                 arguments.environment):
+                                                 arguments.tmplt_file):
                     logger.error("Problem applying ansible playbooks")
     else:
         logger.error(
-            'Unable to read configuration file - ' + arguments.environment)
+            'Unable to read configuration file - ' + arguments.tmplt_file)
         exit(1)
 
     exit(0)
@@ -698,8 +726,11 @@ if __name__ == '__main__':
         default=ARG_NOT_SET,
         help='When cleaning, if this is set, the image will be cleaned too')
     parser.add_argument(
-        '-e', '--env', dest='environment', required=True,
-        help='The environment configuration YAML file - REQUIRED')
+        '-t', '--tmplt', dest='tmplt_file', required=True,
+        help='The SNAPS deployment template YAML file - REQUIRED')
+    parser.add_argument(
+        '-e', '--env-file', dest='env_file',
+        help='Yaml file containing substitution values to the env file')
     parser.add_argument(
         '-l', '--log-level', dest='log_level', default='INFO',
         help='Logging Level (INFO|DEBUG)')
diff --git a/snaps/file_utils.py b/snaps/file_utils.py
index 699d378..a421dd3 100644
--- a/snaps/file_utils.py
+++ b/snaps/file_utils.py
@@ -39,10 +39,11 @@ def file_exists(file_path):
     the path is a directory
     :return:
     """
-    if os.path.exists(file_path):
-        if os.path.isdir(file_path):
+    expanded_path = os.path.expanduser(file_path)
+    if os.path.exists(expanded_path):
+        if os.path.isdir(expanded_path):
             return False
-        return os.path.isfile(file_path)
+        return os.path.isfile(expanded_path)
     return False
 
 
diff --git a/snaps/openstack/create_image.py b/snaps/openstack/create_image.py
index a4c9357..537824d 100644
--- a/snaps/openstack/create_image.py
+++ b/snaps/openstack/create_image.py
@@ -61,8 +61,8 @@ class OpenStackImage:
         if self.__image:
             logger.info('Found image with name - ' + self.image_settings.name)
             return self.__image
-        elif self.image_settings.exists and not self.image_settings.url \
-                and not self.image_settings.image_file:
+        elif (self.image_settings.exists and not self.image_settings.url
+                and not self.image_settings.image_file):
             raise ImageCreationError(
                 'Image with does not exist with name - ' +
                 self.image_settings.name)
@@ -257,8 +257,13 @@ class ImageSettings:
         self.url = kwargs.get('url')
         if not self.url:
             self.url = kwargs.get('download_url')
+        if self.url == 'None':
+            self.url = None
 
         self.image_file = kwargs.get('image_file')
+        if self.image_file == 'None':
+            self.image_file = None
+
         self.extra_properties = kwargs.get('extra_properties')
         self.nic_config_pb_loc = kwargs.get('nic_config_pb_loc')
 
@@ -299,10 +304,6 @@ class ImageSettings:
             raise ImageSettingsError(
                 'URL or image file must be set or image must already exist')
 
-        if self.url and self.image_file:
-            raise ImageSettingsError(
-                'Please set either URL or image file, not both')
-
         if not self.image_user:
             raise ImageSettingsError('Image user is required')
 
diff --git a/snaps/openstack/create_instance.py b/snaps/openstack/create_instance.py
index 6b9a122..2fbeb79 100644
--- a/snaps/openstack/create_instance.py
+++ b/snaps/openstack/create_instance.py
@@ -738,8 +738,12 @@ class VmInstanceSettings:
                                     waiting obtaining an SSH connection to a VM
         :param availability_zone: the name of the compute server on which to
                                   deploy the VM (optional)
-        :param userdata: the cloud-init script to run after the VM has been
-                         started
+        :param userdata: the string contents of any optional cloud-init script
+                         to execute after the VM has been activated.
+                         This value may also contain a dict who's key value
+                         must contain the key 'cloud-init_file' which denotes
+                         the location of some file containing the cloud-init
+                         script
         """
         self.name = kwargs.get('name')
         self.flavor = kwargs.get('flavor')
diff --git a/snaps/openstack/create_keypairs.py b/snaps/openstack/create_keypairs.py
index cc32da3..d0b69cd 100644
--- a/snaps/openstack/create_keypairs.py
+++ b/snaps/openstack/create_keypairs.py
@@ -56,8 +56,12 @@ class OpenStackKeypair:
 
         logger.info('Creating keypair %s...' % self.keypair_settings.name)
 
-        self.__keypair = nova_utils.get_keypair_by_name(
-            self.__nova, self.keypair_settings.name)
+        try:
+            self.__keypair = nova_utils.get_keypair_by_name(
+                self.__nova, self.keypair_settings.name)
+        except Exception as e:
+            logger.warn('Cannot load existing keypair - %s', e)
+            return
 
         if not self.__keypair and not cleanup:
             if self.keypair_settings.public_filepath and os.path.isfile(
@@ -109,13 +113,17 @@ class OpenStackKeypair:
             if (self.keypair_settings.public_filepath and
                     file_utils.file_exists(
                         self.keypair_settings.public_filepath)):
-                os.chmod(self.keypair_settings.public_filepath, 0o777)
-                os.remove(self.keypair_settings.public_filepath)
+                expanded_path = os.path.expanduser(
+                    self.keypair_settings.public_filepath)
+                os.chmod(expanded_path, 0o755)
+                os.remove(expanded_path)
             if (self.keypair_settings.private_filepath and
                     file_utils.file_exists(
                         self.keypair_settings.private_filepath)):
-                os.chmod(self.keypair_settings.private_filepath, 0o777)
-                os.remove(self.keypair_settings.private_filepath)
+                expanded_path = os.path.expanduser(
+                    self.keypair_settings.private_filepath)
+                os.chmod(expanded_path, 0o755)
+                os.remove(expanded_path)
 
     def get_keypair(self):
         """
diff --git a/snaps/openstack/create_project.py b/snaps/openstack/create_project.py
index 7bfdad1..38505ad 100644
--- a/snaps/openstack/create_project.py
+++ b/snaps/openstack/create_project.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 import logging
 
-from keystoneclient.exceptions import NotFound
+from keystoneclient.exceptions import NotFound, Conflict
 
 from snaps.openstack.utils import keystone_utils, neutron_utils, nova_utils
 
@@ -40,6 +40,7 @@ class OpenStackProject:
         self.__project = None
         self.__role = None
         self.__keystone = None
+        self.__role_name = self.project_settings.name + '-role'
 
     def create(self, cleanup=False):
         """
@@ -56,6 +57,14 @@ class OpenStackProject:
         elif not cleanup:
             self.__project = keystone_utils.create_project(
                 self.__keystone, self.project_settings)
+            for username in self.project_settings.users:
+                user = keystone_utils.get_user(self.__keystone, username)
+                if user:
+                    try:
+                        self.assoc_user(user)
+                    except Conflict as e:
+                        logger.warn('Unable to associate user %s due to %s',
+                                    user.name, e)
         else:
             logger.info('Did not create image due to cleanup mode')
 
@@ -93,6 +102,12 @@ class OpenStackProject:
                 pass
             self.__project = None
 
+        # Final role check in case init was done from an existing instance
+        role = keystone_utils.get_role_by_name(
+            self.__keystone, self.__role_name)
+        if role:
+            keystone_utils.delete_role(self.__keystone, role)
+
     def get_project(self):
         """
         Returns the OpenStack project object populated on create()
@@ -107,8 +122,11 @@ class OpenStackProject:
         :return:
         """
         if not self.__role:
-            self.__role = keystone_utils.create_role(
-                self.__keystone, self.project_settings.name + '-role')
+            self.__role = keystone_utils.get_role_by_name(
+                self.__keystone, self.__role_name)
+            if not self.__role:
+                self.__role = keystone_utils.create_role(
+                    self.__keystone, self.__role_name)
 
         keystone_utils.grant_user_role_to_project(self.__keystone, self.__role,
                                                   user, self.__project)
@@ -161,6 +179,7 @@ class ProjectSettings:
                                       (default = 'Default').
                                       Field is used for v3 clients
         :param description: the description (optional)
+        :param users: list of users to associat project to (optional)
         :param enabled: denotes whether or not the user is enabled
                         (default True)
         """
@@ -175,6 +194,8 @@ class ProjectSettings:
         else:
             self.enabled = True
 
+        self.users = kwargs.get('users', list())
+
         if not self.name:
             raise ProjectSettingsError(
                 "The attribute name is required for ProjectSettings")
diff --git a/snaps/openstack/create_security_group.py b/snaps/openstack/create_security_group.py
index 5a0d474..34d5952 100644
--- a/snaps/openstack/create_security_group.py
+++ b/snaps/openstack/create_security_group.py
@@ -15,7 +15,7 @@
 import logging
 
 import enum
-from neutronclient.common.exceptions import NotFound
+from neutronclient.common.exceptions import NotFound, Conflict
 from snaps.openstack.utils import keystone_utils
 from snaps.openstack.utils import neutron_utils
 
@@ -78,9 +78,13 @@ class OpenStackSecurityGroup:
 
             # Create the custom rules
             for sec_grp_rule_setting in self.sec_grp_settings.rule_settings:
-                custom_rule = neutron_utils.create_security_group_rule(
-                    self.__neutron, sec_grp_rule_setting)
-                self.__rules[sec_grp_rule_setting] = custom_rule
+                try:
+                    custom_rule = neutron_utils.create_security_group_rule(
+                        self.__neutron, sec_grp_rule_setting)
+                    self.__rules[sec_grp_rule_setting] = custom_rule
+                except Conflict as e:
+                    logger.warn('Unable to create rule due to conflict - %s',
+                                e)
 
             # Refresh security group object to reflect the new rules added
             self.__security_group = neutron_utils.get_security_group(
@@ -236,6 +240,7 @@ class SecurityGroupSettings:
                 if isinstance(rule_setting, SecurityGroupRuleSettings):
                     self.rule_settings.append(rule_setting)
                 else:
+                    rule_setting['sec_grp_name'] = self.name
                     self.rule_settings.append(SecurityGroupRuleSettings(
                         **rule_setting))
 
diff --git a/snaps/openstack/os_credentials.py b/snaps/openstack/os_credentials.py
index bb68215..6f25237 100644
--- a/snaps/openstack/os_credentials.py
+++ b/snaps/openstack/os_credentials.py
@@ -100,7 +100,7 @@ class OSCreds:
             self.project_domain_name = kwargs['project_domain_name']
 
         if kwargs.get('interface') is None:
-            self.interface = 'admin'
+            self.interface = 'public'
         else:
             self.interface = kwargs['interface']
 
diff --git a/snaps/openstack/tests/conf/os_credentials_tests.py b/snaps/openstack/tests/conf/os_credentials_tests.py
index b63a91d..5efb32c 100644
--- a/snaps/openstack/tests/conf/os_credentials_tests.py
+++ b/snaps/openstack/tests/conf/os_credentials_tests.py
@@ -151,7 +151,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('Default', os_creds.user_domain_name)
         self.assertEqual('default', os_creds.project_domain_id)
         self.assertEqual('Default', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertFalse(os_creds.cacert)
         self.assertIsNone(os_creds.proxy_settings)
         self.assertIsNone(os_creds.region_name)
@@ -172,7 +172,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('Default', os_creds.user_domain_name)
         self.assertEqual('default', os_creds.project_domain_id)
         self.assertEqual('Default', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertFalse(os_creds.cacert)
         self.assertIsNone(os_creds.proxy_settings)
         self.assertIsNone(os_creds.region_name)
@@ -196,7 +196,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('Default', os_creds.user_domain_name)
         self.assertEqual('default', os_creds.project_domain_id)
         self.assertEqual('Default', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertTrue(os_creds.cacert)
         self.assertIsNone(os_creds.proxy_settings)
         self.assertEqual('test_region', os_creds.region_name)
@@ -220,7 +220,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('Default', os_creds.user_domain_name)
         self.assertEqual('default', os_creds.project_domain_id)
         self.assertEqual('Default', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertTrue(os_creds.cacert)
         self.assertIsNone(os_creds.proxy_settings)
         self.assertEqual('test_region', os_creds.region_name)
@@ -242,7 +242,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('Default', os_creds.user_domain_name)
         self.assertEqual('default', os_creds.project_domain_id)
         self.assertEqual('Default', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertFalse(os_creds.cacert)
         self.assertEqual('foo', os_creds.proxy_settings.host)
         self.assertEqual('1234', os_creds.proxy_settings.port)
@@ -270,7 +270,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('domain2', os_creds.user_domain_name)
         self.assertEqual('domain3', os_creds.project_domain_id)
         self.assertEqual('domain4', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertFalse(os_creds.cacert)
         self.assertEqual('foo', os_creds.proxy_settings.host)
         self.assertEqual('1234', os_creds.proxy_settings.port)
@@ -295,7 +295,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('domain2', os_creds.user_domain_name)
         self.assertEqual('domain3', os_creds.project_domain_id)
         self.assertEqual('domain4', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertFalse(os_creds.cacert)
         self.assertEqual('foo', os_creds.proxy_settings.host)
         self.assertEqual('1234', os_creds.proxy_settings.port)
@@ -319,7 +319,7 @@ class OSCredsUnitTests(unittest.TestCase):
         self.assertEqual('Default', os_creds.user_domain_name)
         self.assertEqual('default', os_creds.project_domain_id)
         self.assertEqual('Default', os_creds.project_domain_name)
-        self.assertEqual('admin', os_creds.interface)
+        self.assertEqual('public', os_creds.interface)
         self.assertFalse(os_creds.cacert)
         self.assertEqual('foo', os_creds.proxy_settings.host)
         self.assertEqual('1234', os_creds.proxy_settings.port)
diff --git a/snaps/openstack/tests/create_image_tests.py b/snaps/openstack/tests/create_image_tests.py
index 7a6db86..f70a71c 100644
--- a/snaps/openstack/tests/create_image_tests.py
+++ b/snaps/openstack/tests/create_image_tests.py
@@ -77,19 +77,6 @@ class ImageSettingsUnitTests(unittest.TestCase):
             ImageSettings(
                 **{'name': 'foo', 'image_user': 'bar', 'format': 'qcow2'})
 
-    def test_name_user_format_url_file_only(self):
-        with self.assertRaises(ImageSettingsError):
-            ImageSettings(name='foo', image_user='bar', img_format='qcow2',
-                          url='http://foo.com',
-                          image_file='/foo/bar.qcow')
-
-    def test_config_with_name_user_format_url_file_only(self):
-        with self.assertRaises(ImageSettingsError):
-            ImageSettings(
-                **{'name': 'foo', 'image_user': 'bar', 'format': 'qcow2',
-                   'download_url': 'http://foo.com',
-                   'image_file': '/foo/bar.qcow'})
-
     def test_name_user_format_url_only(self):
         settings = ImageSettings(name='foo', image_user='bar',
                                  img_format='qcow2', url='http://foo.com')
diff --git a/snaps/openstack/tests/create_project_tests.py b/snaps/openstack/tests/create_project_tests.py
index 0e1d0ae..aa9dcfb 100644
--- a/snaps/openstack/tests/create_project_tests.py
+++ b/snaps/openstack/tests/create_project_tests.py
@@ -49,6 +49,7 @@ class ProjectSettingsUnitTests(unittest.TestCase):
         self.assertEqual('Default', settings.domain_name)
         self.assertIsNone(settings.description)
         self.assertTrue(settings.enabled)
+        self.assertEqual(list(), settings.users)
 
     def test_config_with_name_only(self):
         settings = ProjectSettings(**{'name': 'foo'})
@@ -56,23 +57,29 @@ class ProjectSettingsUnitTests(unittest.TestCase):
         self.assertEqual('Default', settings.domain_name)
         self.assertIsNone(settings.description)
         self.assertTrue(settings.enabled)
+        self.assertEqual(list(), settings.users)
 
     def test_all(self):
-        settings = ProjectSettings(name='foo', domain='bar',
-                                   description='foobar', enabled=False)
+        users = ['test1', 'test2']
+        settings = ProjectSettings(
+            name='foo', domain='bar', description='foobar', enabled=False,
+            users=users)
         self.assertEqual('foo', settings.name)
         self.assertEqual('bar', settings.domain_name)
         self.assertEqual('foobar', settings.description)
         self.assertFalse(settings.enabled)
+        self.assertEqual(users, settings.users)
 
     def test_config_all(self):
+        users = ['test1', 'test2']
         settings = ProjectSettings(
             **{'name': 'foo', 'domain': 'bar', 'description': 'foobar',
-               'enabled': False})
+               'enabled': False, 'users': users})
         self.assertEqual('foo', settings.name)
         self.assertEqual('bar', settings.domain_name)
         self.assertEqual('foobar', settings.description)
         self.assertFalse(settings.enabled)
+        self.assertEqual(users, settings.users)
 
 
 class CreateProjectSuccessTests(OSComponentTestCase):
diff --git a/snaps/openstack/utils/glance_utils.py b/snaps/openstack/utils/glance_utils.py
index 2606e32..a127ad3 100644
--- a/snaps/openstack/utils/glance_utils.py
+++ b/snaps/openstack/utils/glance_utils.py
@@ -168,7 +168,7 @@ def __create_image_v2(glance, image_settings):
     """
     cleanup_temp_file = False
     image_file = None
-    if image_settings.image_file:
+    if image_settings.image_file is not None:
         image_filename = image_settings.image_file
     elif image_settings.url:
         file_name = str(uuid.uuid4())
@@ -199,7 +199,7 @@ def __create_image_v2(glance, image_settings):
             kwargs.update(image_settings.extra_properties)
 
         os_image = glance.images.create(**kwargs)
-        image_file = open(image_filename, 'rb')
+        image_file = open(os.path.expanduser(image_filename), 'rb')
         glance.images.upload(os_image['id'], image_file)
     except:
         logger.error('Unexpected exception creating image. Rolling back')
diff --git a/snaps/openstack/utils/keystone_utils.py b/snaps/openstack/utils/keystone_utils.py
index b36c19f..46f6fb8 100644
--- a/snaps/openstack/utils/keystone_utils.py
+++ b/snaps/openstack/utils/keystone_utils.py
@@ -176,6 +176,7 @@ def create_project(keystone, project_settings):
             enabled=project_settings.enabled)
         domain_id = os_project.domain_id
 
+    logger.info('Created project with name - %s', project_settings.name)
     return Project(
         name=os_project.name, project_id=os_project.id, domain_id=domain_id)
 
@@ -186,6 +187,7 @@ def delete_project(keystone, project):
     :param keystone: the Keystone clien
     :param project: the SNAPS-OO Project domain object
     """
+    logger.info('Deleting project with name - %s', project.name)
     if keystone.version == V2_VERSION_STR:
         keystone.tenants.delete(project.id)
     else:
@@ -273,6 +275,7 @@ def create_user(keystone, user_settings):
                     project=os_project)
 
     if os_user:
+        logger.info('Created user with name - %s', os_user.name)
         return User(name=os_user.name, user_id=os_user.id)
 
 
@@ -282,6 +285,7 @@ def delete_user(keystone, user):
     :param keystone: the Keystone client
     :param user: the SNAPS-OO User domain object
     """
+    logger.info('Deleting user with name - %s', user.name)
     keystone.users.delete(user.id)
 
 
@@ -337,6 +341,7 @@ def create_role(keystone, name):
     :return: a SNAPS-OO Role domain object
     """
     role = keystone.roles.create(name)
+    logger.info('Created role with name - %s', role.name)
     return Role(name=role.name, role_id=role.id)
 
 
@@ -347,6 +352,7 @@ def delete_role(keystone, role):
     :param role: the SNAPS-OO Role domain object to delete
     :return:
     """
+    logger.info('Deleting role with name - %s', role.name)
     keystone.roles.delete(role.id)
 
 
@@ -361,6 +367,7 @@ def grant_user_role_to_project(keystone, role, user, project):
     """
 
     os_role = get_role_by_id(keystone, role.id)
+    logger.info('Granting role %s to project %s', role.name, project)
     if keystone.version == V2_VERSION_STR:
         keystone.roles.add_user_role(user, os_role, tenant=project)
     else:
diff --git a/snaps/openstack/utils/nova_utils.py b/snaps/openstack/utils/nova_utils.py
index fe53211..1665fd0 100644
--- a/snaps/openstack/utils/nova_utils.py
+++ b/snaps/openstack/utils/nova_utils.py
@@ -22,6 +22,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
 from novaclient.client import Client
 from novaclient.exceptions import NotFound
 
+from snaps import file_utils
 from snaps.domain.flavor import Flavor
 from snaps.domain.keypair import Keypair
 from snaps.domain.project import ComputeQuotas
@@ -86,6 +87,18 @@ def create_server(nova, neutron, glance, instance_settings, image_settings,
 
     image = glance_utils.get_image(glance, image_settings=image_settings)
     if image:
+        userdata = None
+        if instance_settings.userdata:
+            if isinstance(instance_settings.userdata, str):
+                userdata = instance_settings.userdata + '\n'
+            elif (isinstance(instance_settings.userdata, dict) and
+                  'script_file' in instance_settings.userdata):
+                try:
+                    userdata = file_utils.read_file(
+                        instance_settings.userdata['script_file'])
+                except Exception as e:
+                    logger.warn('error reading userdata file %s - %s',
+                                instance_settings.userdata, e)
         args = {'name': instance_settings.name,
                 'flavor': flavor,
                 'image': image,
@@ -93,7 +106,7 @@ def create_server(nova, neutron, glance, instance_settings, image_settings,
                 'key_name': keypair_name,
                 'security_groups':
                     instance_settings.security_group_names,
-                'userdata': instance_settings.userdata}
+                'userdata': userdata}
 
         if instance_settings.availability_zone:
             args['availability_zone'] = instance_settings.availability_zone
@@ -264,6 +277,58 @@ def public_key_openssh(keys):
                                           serialization.PublicFormat.OpenSSH)
 
 
+def save_keys_to_files(keys=None, pub_file_path=None, priv_file_path=None):
+    """
+    Saves the generated RSA generated keys to the filesystem
+    :param keys: the keys to save generated by cryptography
+    :param pub_file_path: the path to the public keys
+    :param priv_file_path: the path to the private keys
+    """
+    if keys:
+        if pub_file_path:
+            # To support '~'
+            pub_expand_file = os.path.expanduser(pub_file_path)
+            pub_dir = os.path.dirname(pub_expand_file)
+
+            if not os.path.isdir(pub_dir):
+                os.mkdir(pub_dir)
+
+            public_handle = None
+            try:
+                public_handle = open(pub_expand_file, 'wb')
+                public_bytes = keys.public_key().public_bytes(
+                    serialization.Encoding.OpenSSH,
+                    serialization.PublicFormat.OpenSSH)
+                public_handle.write(public_bytes)
+            finally:
+                if public_handle:
+                    public_handle.close()
+
+            os.chmod(pub_expand_file, 0o600)
+            logger.info("Saved public key to - " + pub_expand_file)
+        if priv_file_path:
+            # To support '~'
+            priv_expand_file = os.path.expanduser(priv_file_path)
+            priv_dir = os.path.dirname(priv_expand_file)
+            if not os.path.isdir(priv_dir):
+                os.mkdir(priv_dir)
+
+            private_handle = None
+            try:
+                private_handle = open(priv_expand_file, 'wb')
+                private_handle.write(
+                    keys.private_bytes(
+                        encoding=serialization.Encoding.PEM,
+                        format=serialization.PrivateFormat.TraditionalOpenSSL,
+                        encryption_algorithm=serialization.NoEncryption()))
+            finally:
+                if private_handle:
+                    private_handle.close()
+
+            os.chmod(priv_expand_file, 0o600)
+            logger.info("Saved private key to - " + priv_expand_file)
+
+
 def upload_keypair_file(nova, name, file_path):
     """
     Uploads a public key from a file