#!/usr/bin/python
#
# Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
# and others. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is responsible for deploying virtual environments
import argparse
import logging
import re
import time
from jinja2 import Environment, FileSystemLoader
import os
import yaml
from snaps import file_utils
from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
from snaps.openstack.create_image import ImageSettings, OpenStackImage
from snaps.openstack.create_instance import VmInstanceSettings
from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
from snaps.openstack.create_network import (
PortSettings, NetworkSettings, OpenStackNetwork)
from snaps.openstack.create_project import OpenStackProject, ProjectSettings
from snaps.openstack.create_qos import QoSSettings, OpenStackQoS
from snaps.openstack.create_router import RouterSettings, OpenStackRouter
from snaps.openstack.create_security_group import (
OpenStackSecurityGroup, SecurityGroupSettings)
from snaps.openstack.create_user import OpenStackUser, UserSettings
from snaps.openstack.create_volume import OpenStackVolume, VolumeSettings
from snaps.openstack.create_volume_type import (
OpenStackVolumeType, VolumeTypeSettings)
from snaps.openstack.os_credentials import OSCreds, ProxySettings
from snaps.openstack.utils import deploy_utils
from snaps.provisioning import ansible_utils
__author__ = 'spisarski'
logger = logging.getLogger('snaps_launcher')
ARG_NOT_SET = "argument not set"
DEFAULT_CREDS_KEY = 'admin'
def __get_creds_dict(os_conn_config):
"""
Returns a dict of OSCreds where the key is the creds name.
For backwards compatibility, credentials not contained in a list (only
one) will be returned with the key of None
:param os_conn_config: the credential configuration
:return: a dict of OSCreds objects
"""
if 'connection' in os_conn_config:
return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
elif 'connections' in os_conn_config:
out = dict()
for os_conn_dict in os_conn_config['connections']:
config = os_conn_dict.get('connection')
if not config:
raise Exception('Invalid connection format')
name = config.get('name')
if not name:
raise Exception('Connection config requires a name field')
out[name] = __get_os_credentials(os_conn_dict)
return out
def __get_creds(os_creds_dict, os_user_dict, inst_config):
"""
Returns the appropriate credentials
:param os_creds_dict: a dictionary of OSCreds objects where the name is the
key
:param os_user_dict: a dictionary of OpenStackUser objects where the name
is the key
:param inst_config:
:return: an OSCreds instance or None
"""
os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
if 'os_user' in inst_config:
os_user_conf = inst_config['os_user']
if 'name' in os_user_conf:
user_creator = os_user_dict.get(os_user_conf['name'])
if user_creator:
return user_creator.get_os_creds(
project_name=os_user_conf.get('project_name'))
elif 'os_creds_name' in inst_config:
if 'os_creds_name' in inst_config:
os_creds = os_creds_dict[inst_config['os_creds_name']]
return os_creds
def __get_os_credentials(os_conn_config):
"""
Returns an object containing all of the information required to access
OpenStack APIs
:param os_conn_config: The configuration holding the credentials
:return: an OSCreds instance
"""
config = os_conn_config.get('connection')
if not config:
raise Exception('Invalid connection configuration')
proxy_settings = None
http_proxy = config.get('http_proxy')
if http_proxy:
tokens = re.split(':', http_proxy)
ssh_proxy_cmd = config.get('ssh_proxy_cmd')
proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
ssh_proxy_cmd=ssh_proxy_cmd)
else:
if 'proxy_settings' in config:
host = config['proxy_settings'].get('host')
port = config['proxy_settings'].get('port')
if host and host != 'None' and port and port != 'None':
proxy_settings = ProxySettings(**config['proxy_settings'])
if proxy_settings:
config['proxy_settings'] = proxy_settings
else:
if config.get('proxy_settings'):
del config['proxy_settings']
return OSCreds(**config)
def __parse_ports_config(config):
"""
Parses the "ports" configuration
:param config: The dictionary to parse
:return: a list of PortConfig objects
"""
out = list()
for port_config in config:
out.append(PortSettings(**port_config.get('port')))
return out
def __create_instances(os_creds_dict, creator_class, config_class, config,
config_key, cleanup=False, os_users_dict=None):
"""
Returns a dictionary of SNAPS creator objects where the key is the name
:param os_creds_dict: Dictionary of OSCreds objects where the key is the
name
:param config: The list of configurations for the same type
:param config_key: The list of configurations for the same type
:param cleanup: Denotes whether or not this is being called for cleanup
:return: dictionary
"""
out = {}
if config:
try:
for config_dict in config:
inst_config = config_dict.get(config_key)
if inst_config:
creator = creator_class(
__get_creds(os_creds_dict, os_users_dict, inst_config),
config_class(**inst_config))
if cleanup:
creator.initialize()
else:
creator.create()
out[inst_config['name']] = creator
logger.info('Created configured %s', config_key)
except Exception as e:
logger.error('Unexpected error instantiating creator [%s] '
'with exception %s', creator_class, e)
return out
def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
image_dict, keypairs_dict, cleanup=False):
"""
Returns a dictionary of OpenStackVmInstance objects where the key is the
instance name
:param os_creds_dict: Dictionary of OSCreds objects where the key is the
name
:param os_users_dict: Dictionary of OpenStackUser objects where the key is
the username
:param instances_config: The list of VM instance configurations
:param image_dict: A dictionary of images that will probably be used to
instantiate the VM instance
:param keypairs_dict: A dictionary of keypairs that will probably be used
to instantiate the VM instance
:param cleanup: Denotes whether or not this is being called for cleanup
:return: dictionary
"""
vm_dict = {}
if instances_config:
try:
for instance_config in instances_config:
conf = instance_config.get('instance')
if conf:
if image_dict:
image_creator = image_dict.get(conf.get('imageName'))
if image_creator:
instance_settings = VmInstanceSettings(
**instance_config['instance'])
kp_creator = keypairs_dict.get(
conf.get('keypair_name'))
vm_dict[conf[
'name']] = deploy_utils.create_vm_instance(
__get_creds(
os_creds_dict, os_users_dict, conf),
instance_settings,
image_creator.image_settings,
keypair_creator=kp_creator,
init_only=cleanup)
else:
raise Exception('Image creator instance not found.'
' Cannot instantiate')
else:
raise Exception('Image dictionary is None. Cannot '
'instantiate')
else:
raise Exception('Instance configuration is None. Cannot '
'instantiate')
logger.info('Created configured instances')
except Exception as e:
logger.error('Unexpected error creating VM instances - %s', e)
return vm_dict
def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
image_dict, flavor_dict, env_file):
"""
Applies ansible playbooks to running VMs with floating IPs
:param ansible_configs: a list of Ansible configurations
:param os_creds_dict: Dictionary of OSCreds objects where the key is the
name
:param vm_dict: the dictionary of newly instantiated VMs where the name is
the key
:param image_dict: the dictionary of newly instantiated images where the
name is the key
:param flavor_dict: the dictionary of newly instantiated flavors where the
name is the key
:param env_file: the path of the environment for setting the CWD so
playbook location is relative to the deployment file
:return: t/f - true if successful
"""
logger.info("Applying Ansible Playbooks")
if ansible_configs:
# Ensure all hosts are accepting SSH session requests
for vm_inst in list(vm_dict.values()):
if not vm_inst.vm_ssh_active(block=True):
logger.warning(
"Timeout waiting for instance to respond to SSH requests")
return False
# Set CWD so the deployment file's playbook location can leverage
# relative paths
orig_cwd = os.getcwd()
env_dir = os.path.dirname(env_file)
os.chdir(env_dir)
# Apply playbooks
for ansible_config in ansible_configs:
if 'pre_sleep_time' in ansible_config:
try:
sleep_time = int(ansible_config['pre_sleep_time'])
logger.info('Waiting %s seconds to apply playbooks',
sleep_time)
time.sleep(sleep_time)
except:
pass
os_creds = os_creds_dict.get(None, 'admin')
__apply_ansible_playbook(ansible_config, os_creds, vm_dict,
image_dict, flavor_dict)
# Return to original directory
os.chdir(orig_cwd)
return True
def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
flavor_dict):
"""
Applies an Ansible configuration setting
:param ansible_config: the configuration settings
:param os_creds: the OpenStack credentials object
:param vm_dict: the dictionary of newly instantiated VMs where the name is
the key
:param image_dict: the dictionary of newly instantiated images where the
name is the key
:param flavor_dict: the dictionary of newly instantiated flavors where the
name is the key
"""
if ansible_config:
(remote_user, floating_ips, private_key_filepath,
proxy_settings) = __get_connection_info(
ansible_config, vm_dict)
if floating_ips:
retval = ansible_utils.apply_playbook(
ansible_config['playbook_location'], floating_ips, remote_user,
private_key_filepath,
variables=__get_variables(ansible_config.get('variables'),
os_creds, vm_dict, image_dict,
flavor_dict),
proxy_setting=proxy_settings)
if retval != 0:
# Not a fatal type of event
logger.warning(
'Unable to apply playbook found at location - %s',
ansible_config.get('playbook_location'))
def __get_connection_info(ansible_config, vm_dict):
"""
Returns a tuple of data required for connecting to the running VMs
(remote_user, [floating_ips], private_key_filepath, proxy_settings)
:param ansible_config: the configuration settings
:param vm_dict: the dictionary of VMs where the VM name is the key
:return: tuple where the first element is the user and the second is a list
of floating IPs and the third is the
private key file location and the fourth is an instance of the
snaps.ProxySettings class
(note: in order to work, each of the hosts need to have the same sudo_user
and private key file location values)
"""
if ansible_config.get('hosts'):
hosts = ansible_config['hosts']
if len(hosts) > 0:
floating_ips = list()
remote_user = None
pk_file = None
proxy_settings = None
for host in hosts:
vm = vm_dict.get(host)
if vm:
fip = vm.get_floating_ip()
if fip:
remote_user = vm.get_image_user()
if fip:
floating_ips.append(fip.ip)
else:
raise Exception(
'Could not find floating IP for VM - ' +
vm.name)
pk_file = vm.keypair_settings.private_filepath
proxy_settings = vm.get_os_creds().proxy_settings
else:
logger.error('Could not locate VM with name - ' + host)
return remote_user, floating_ips, pk_file, proxy_settings
return None
def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict):
"""
Returns a dictionary of substitution variables to be used for Ansible
templates
:param var_config: the variable configuration settings
:param os_creds: the OpenStack credentials object
:param vm_dict: the dictionary of newly instantiated VMs where the name is
the key
:param image_dict: the dictionary of newly instantiated images where the
name is the key
:param flavor_dict: the dictionary of newly instantiated flavors where the
name is the key
:return: dictionary or None
"""
if var_config and vm_dict and len(vm_dict) > 0:
variables = dict()
for key, value in var_config.items():
value = __get_variable_value(value, os_creds, vm_dict, image_dict,
flavor_dict)
if key and value:
variables[key] = value
logger.info(
"Set Jinga2 variable with key [%s] the value [%s]",
key, value)
else:
logger.warning('Key [%s] or Value [%s] must not be None',
str(key), str(value))
return variables
return None
def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
flavor_dict):
"""
Returns the associated variable value for use by Ansible for substitution
purposes
:param var_config_values: the configuration dictionary
:param os_creds: the OpenStack credentials object
:param vm_dict: the dictionary of newly instantiated VMs where the name is
the key
:param image_dict: the dictionary of newly instantiated images where the
name is the key
:param flavor_dict: the dictionary of newly instantiated flavors where the
name is the key
:return:
"""
if var_config_values['type'] == 'string':
return __get_string_variable_value(var_config_values)
if var_config_values['type'] == 'vm-attr':
return __get_vm_attr_variable_value(var_config_values, vm_dict)
if var_config_values['type'] == 'os_creds':
return __get_os_creds_variable_value(var_config_values, os_creds)
if var_config_values['type'] == 'port':
return __get_vm_port_variable_value(var_config_values, vm_dict)
if var_config_values['type'] == 'floating_ip':
return __get_vm_fip_variable_value(var_config_values, vm_dict)
if var_config_values['type'] == 'image':
return __get_image_variable_value(var_config_values, image_dict)
if var_config_values['type'] == 'flavor':
return __get_flavor_variable_value(var_config_values, flavor_dict)
return None
def __get_string_variable_value(var_config_values):
"""
Returns the associated string value
:param var_config_values: the configuration dictionary
:return: the value contained in the dictionary with the key 'value'
"""
return var_config_values['value']
def __get_vm_attr_variable_value(var_config_values, vm_dict):
"""
Returns the associated value contained on a VM instance
:param var_config_values: the configuration dictionary
:param vm_dict: the dictionary containing all VMs where the key is the VM's
name
:return: the value
"""
vm = vm_dict.get(var_config_values['vm_name'])
if vm:
if var_config_values['value'] == 'floating_ip':
return vm.get_floating_ip().ip
if var_config_values['value'] == 'image_user':
return vm.get_image_user()
def __get_os_creds_variable_value(var_config_values, os_creds):
"""
Returns the associated OS credentials value
:param var_config_values: the configuration dictionary
:param os_creds: the credentials
:return: the value
"""
logger.info("Retrieving OS Credentials")
if os_creds:
if var_config_values['value'] == 'username':
logger.info("Returning OS username")
return os_creds.username
elif var_config_values['value'] == 'password':
logger.info("Returning OS password")
return os_creds.password
elif var_config_values['value'] == 'auth_url':
logger.info("Returning OS auth_url")
return os_creds.auth_url
elif var_config_values['value'] == 'project_name':
logger.info("Returning OS project_name")
return os_creds.project_name
logger.info("Returning none")
return None
def __get_vm_port_variable_value(var_config_values, vm_dict):
"""
Returns the associated OS credentials value
:param var_config_values: the configuration dictionary
:param vm_dict: the dictionary containing all VMs where the key is the VM's
name
:return: the value
"""
port_name = var_config_values.get('port_name')
vm_name = var_config_values.get('vm_name')
if port_name and vm_name:
vm = vm_dict.get(vm_name)
if vm:
port_value_id = var_config_values.get('port_value')
if port_value_id:
if port_value_id == 'mac_address':
return vm.get_port_mac(port_name)
if port_value_id == 'ip_address':
return vm.get_port_ip(port_name)
def __get_vm_fip_variable_value(var_config_values, vm_dict):
"""
Returns the floating IP value if found
:param var_config_values: the configuration dictionary
:param vm_dict: the dictionary containing all VMs where the key is the VM's
name
:return: the floating IP string value or None
"""
fip_name = var_config_values.get('fip_name')
vm_name = var_config_values.get('vm_name')
if vm_name:
vm = vm_dict.get(vm_name)
if vm:
fip = vm.get_floating_ip(fip_name)
if fip:
return fip.ip
def __get_image_variable_value(var_config_values, image_dict):
"""
Returns the associated image value
:param var_config_values: the configuration dictionary
:param image_dict: the dictionary containing all images where the key is
the name
:return: the value
"""
logger.info("Retrieving image values")
if image_dict:
if var_config_values.get('image_name'):
image_creator = image_dict.get(var_config_values['image_name'])
if image_creator:
if var_config_values.get('value') and \
var_config_values['value'] == 'id':
return image_creator.get_image().id
if var_config_values.get('value') and \
var_config_values['value'] == 'user':
return image_creator.image_settings.image_user
logger.info("Returning none")
return None
def __get_flavor_variable_value(var_config_values, flavor_dict):
"""
Returns the associated flavor value
:param var_config_values: the configuration dictionary
:param flavor_dict: the dictionary containing all flavor creators where the
key is the name
:return: the value or None
"""
logger.info("Retrieving flavor values")
if flavor_dict:
if var_config_values.get('flavor_name'):
flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
if flavor_creator:
if var_config_values.get('value') and \
var_config_values['value'] == 'id':
return flavor_creator.get_flavor().id
def main(arguments):
"""
Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or
Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg
containing the following content:
[defaults]
host_key_checking = False
CWD must be this directory where this script is located.
:return: To the OS
"""
log_level = logging.INFO
if arguments.log_level != 'INFO':
log_level = logging.DEBUG
logging.basicConfig(level=log_level)
logger.info('Starting to Deploy')
# Apply env_file/substitution file to template
env = Environment(loader=FileSystemLoader(
searchpath=os.path.dirname(arguments.tmplt_file)))
template = env.get_template(os.path.basename(arguments.tmplt_file))
env_dict = dict()
if arguments.env_file:
env_dict = file_utils.read_yaml(arguments.env_file)
output = template.render(**env_dict)
config = yaml.load(output)
if config:
os_config = config.get('openstack')
creators = list()
vm_dict = dict()
images_dict = dict()
flavors_dict = dict()
os_creds_dict = dict()
clean = arguments.clean is not ARG_NOT_SET
if os_config:
os_creds_dict = __get_creds_dict(os_config)
try:
# Create projects
projects_dict = __create_instances(
os_creds_dict, OpenStackProject, ProjectSettings,
os_config.get('projects'), 'project', clean)
creators.append(projects_dict)
# Create users
users_dict = __create_instances(
os_creds_dict, OpenStackUser, UserSettings,
os_config.get('users'), 'user', clean)
creators.append(users_dict)
# Associate new users to projects
if not clean:
for project_creator in projects_dict.values():
users = project_creator.project_settings.users
for user_name in users:
user_creator = users_dict.get(user_name)
if user_creator:
project_creator.assoc_user(
user_creator.get_user())
# Create flavors
flavors_dict = __create_instances(
os_creds_dict, OpenStackFlavor, FlavorSettings,
os_config.get('flavors'), 'flavor', clean, users_dict)
creators.append(flavors_dict)
# Create QoS specs
qos_dict = __create_instances(
os_creds_dict, OpenStackQoS, QoSSettings,
os_config.get('qos_specs'), 'qos_spec', clean, users_dict)
creators.append(qos_dict)
# Create volume types
vol_type_dict = __create_instances(
os_creds_dict, OpenStackVolumeType, VolumeTypeSettings,
os_config.get('volume_types'), 'volume_type', clean,
users_dict)
creators.append(vol_type_dict)
# Create volume types
vol_dict = __create_instances(
os_creds_dict, OpenStackVolume, VolumeSettings,
os_config.get('volumes'), 'volume', clean, users_dict)
creators.append(vol_dict)
# Create images
images_dict = __create_instances(
os_creds_dict, OpenStackImage, ImageSettings,
os_config.get('images'), 'image', clean, users_dict)
creators.append(images_dict)
# Create networks
creators.append(__create_instances(
os_creds_dict, OpenStackNetwork, NetworkSettings,
os_config.get('networks'), 'network', clean, users_dict))
# Create routers
creators.append(__create_instances(
os_creds_dict, OpenStackRouter, RouterSettings,
os_config.get('routers'), 'router', clean, users_dict))
# Create keypairs
keypairs_dict = __create_instances(
os_creds_dict, OpenStackKeypair, KeypairSettings,
os_config.get('keypairs'), 'keypair', clean, users_dict)
creators.append(keypairs_dict)
# Create security groups
creators.append(__create_instances(
os_creds_dict, OpenStackSecurityGroup,
SecurityGroupSettings,
os_config.get('security_groups'), 'security_group', clean,
users_dict))
# Create instance
vm_dict = __create_vm_instances(
os_creds_dict, users_dict, os_config.get('instances'),
images_dict, keypairs_dict,
arguments.clean is not ARG_NOT_SET)
creators.append(vm_dict)
logger.info(
'Completed creating/retrieving all configured instances')
except Exception as e:
logger.error(
'Unexpected error deploying environment. Rolling back due'
' to - ' + str(e))
raise
# Must enter either block
if arguments.clean is not ARG_NOT_SET:
# Cleanup Environment
__cleanup(creators, arguments.clean_image is not ARG_NOT_SET)
elif arguments.deploy is not ARG_NOT_SET:
logger.info('Configuring NICs where required')
for vm in vm_dict.values():
vm.config_nics()
logger.info('Completed NIC configuration')
# Provision VMs
ansible_config = config.get('ansible')
if ansible_config and vm_dict:
if not __apply_ansible_playbooks(ansible_config,
os_creds_dict, vm_dict,
images_dict, flavors_dict,
arguments.tmplt_file):
logger.error("Problem applying ansible playbooks")
else:
logger.error(
'Unable to read configuration file - ' + arguments.tmplt_file)
exit(1)
exit(0)
def __cleanup(creators, clean_image=False):
for creator_dict in reversed(creators):
for key, creator in creator_dict.items():
if ((isinstance(creator, OpenStackImage) and clean_image)
or not isinstance(creator, OpenStackImage)):
try:
creator.clean()
except Exception as e:
logger.warning('Error cleaning component - %s', e)
if __name__ == '__main__':
# To ensure any files referenced via a relative path will begin from the
# directory in which this file resides
os.chdir(os.path.dirname(os.path.realpath(__file__)))
parser = argparse.ArgumentParser()
parser.add_argument(
'-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET,
help='When used, environment will be deployed and provisioned')
parser.add_argument(
'-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET,
help='When used, the environment will be removed')
parser.add_argument(
'-i', '--clean-image', dest='clean_image', nargs='?',
default=ARG_NOT_SET,
help='When cleaning, if this is set, the image will be cleaned too')
parser.add_argument(
'-t', '--tmplt', dest='tmplt_file', required=True,
help='The SNAPS deployment template YAML file - REQUIRED')
parser.add_argument(
'-e', '--env-file', dest='env_file',
help='Yaml file containing substitution values to the env file')
parser.add_argument(
'-l', '--log-level', dest='log_level', default='INFO',
help='Logging Level (INFO|DEBUG)')
args = parser.parse_args()
if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET:
print(
'Must enter either -d for deploy or -c for cleaning up and '
'environment')
exit(1)
if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET:
print('Cannot enter both options -d/--deploy and -c/--clean')
exit(1)
main(args)