From 406584267b076f60c54d6cfef518dc22a82ae76c Mon Sep 17 00:00:00 2001 From: randyl Date: Fri, 3 Mar 2017 15:30:05 -0700 Subject: Fixed docs build and examples Modified index.rst to include the other docs. Removed redundnat items. Cleaned up the examples. JIRA: SNAPS-41 Change-Id: I7058f1604135f6a2af0376d7b3d2ebc7bafd63e7 Signed-off-by: randyl --- docs/how-to-use/InstallSnaps.rst | 44 ++ docs/how-to-use/LibraryUsage.rst | 503 ++++++++++++++++ docs/how-to-use/Testing.rst | 31 +- docs/how-to-use/VirtEnvDeploy.rst | 61 +- docs/how-to-use/index.rst | 33 +- .../complex-network/deploy-complex-network.yaml | 4 +- examples/external-network/deploy-ext-net.yaml | 4 +- examples/launch.py | 635 +++++++++++++++++++++ examples/simple/deploy-simple.yaml | 3 +- examples/two-network/deploy-two-net-centos.yaml | 4 +- examples/two-network/deploy-two-net-ubuntu.yaml | 4 +- snaps/deploy_venv.py | 635 --------------------- 12 files changed, 1233 insertions(+), 728 deletions(-) create mode 100644 docs/how-to-use/InstallSnaps.rst create mode 100644 docs/how-to-use/LibraryUsage.rst create mode 100644 examples/launch.py delete mode 100644 snaps/deploy_venv.py diff --git a/docs/how-to-use/InstallSnaps.rst b/docs/how-to-use/InstallSnaps.rst new file mode 100644 index 0000000..9412579 --- /dev/null +++ b/docs/how-to-use/InstallSnaps.rst @@ -0,0 +1,44 @@ +**************** +Installing SNAPS +**************** + + +Install Dependencies +==================== +A few packages need to installed onto your system, before you can install SNAPS. + +Git is used to download the snaps source from the OPNFV Gerrit repository. + +Python, GCC and additional libraries are required to compile and install the packages used by SNAPS. These +dependencies need to be installed whether or not a virtual Python environment is used. + +CentOS 7 +-------- + +:: + + # yum install -7 git gcc python-pip python-devel openssl-devel + +Ubuntu +------ +:: + + # apt-get install git python2.7-dev libssl-dev + +Optional: Setup a Python virtual environment +-------------------------------------------- + +Python 2.7 (recommend leveraging a Virtual Python runtime, e.g. + `Virtualenv `__, in your development + environment) + +Install SNAPS dependencies +-------------------------- + +The "pip" command below needs to be executed as root, if you are not using a virtual Python environment. + +:: + + # pip install -e /snaps/ + +The install should now be complete and you can start using the SNAPS-OO libraries. diff --git a/docs/how-to-use/LibraryUsage.rst b/docs/how-to-use/LibraryUsage.rst new file mode 100644 index 0000000..6c84afd --- /dev/null +++ b/docs/how-to-use/LibraryUsage.rst @@ -0,0 +1,503 @@ +********************** +SNAPS-OO Library Usage +********************** + +The pattern used within the SNAPS-OO library for creating OpenStack +instances have been made as consistent as possible amongst the different +instance types. Each consists of a constructor that takes in a +credentials object and generally takes in a single "settings" +(configuration) object. The only exception to this rule is with the +OpenStackVMInstance (creates an OpenStack Server) where it takes in the +additional settings used for the associated image and SSH key-pairs +credentials as those objects contain additional attributes required of +SNAPS, primarily when one needs to obtain remote access. After +instantiation, the create() method must be called to initiate all of the +necessary remote API calls to OpenStack required for proper instance +creation. + +SNAPS Credentials +================= + +As communicating with OpenStack is performed via secure remote RESTful +API calls, any function or method performing any type of query or CRUD +operation must know how to connect to the NFVI. The class ***OSCreds*** +defined in *snaps.openstack.os\_credentials.py* contains everything +required to connect to any Keystone v2.0 or v3 authorization server. The +attributes are listed below: + +- username +- password +- auth\_url +- project\_name (aka. tenant\_name) +- identity\_api\_version (for obtaining Keystone authorization token. + Versions 2.0 & v3 only validated.) +- image\_api\_version (Glance version 1 currently only validated) +- network\_api\_version (Neutron version 2 currently only validated) +- compute\_api\_version (Nova version 2 currently only validated) +- user\_domain\_id (default='default') +- project\_domain\_id (default='default') +- proxy\_settings + + - host (the HTTP proxy host) + - port (the HTTP proxy port) + - ssh\_proxy\_cmd (same as the value placed into ssh -o + ProxyCommand='') + +Create OS Credentials Object +---------------------------- + +.. code:: python + + from snaps.openstack.os_credentials import OSCreds + os_creds=OSCreds(username='admin', password`='admin' + auth_url='http://localhost:5000/v3, project_name='admin', + identity_api_version=3) + +SNAPS Object Creators +===================== + +Each creator minimally requires an OSCreds object for connecting to the +NFVI, associated \*Settings object for instance configuration, create() +method to make the necessary remote API calls and create all of the +necessary OpenStack instances required, and clean() method that is +responsible for deleting all associated OpenStack instances. Please see +the class diagram `here `__. Below is a +textual representation of the requirements of each creator classes with +their associated setting classes and a sample code snippet on how to use +the code. + +Create User +----------- +- User - snaps.openstack.create\_user.OpenStackUser + + - snaps.openstack.create\_user.UserSettings + + - name - the username (required) + - password - the user's password (required) + - project\_name - the name of the project to associated to this + user (optional) + - domain\_name - the user's domain (default='default') + - email - the user's email address (optional) + - enabled - flag to determine whether or not the user should be + enabled (default=True) + +.. code:: python + + from snaps.openstack.create_user import UserSettings, OpenStackUser + user_settings = UserSettings(name='username', password='password) + user_creator = OpenStackUser(os_creds, user_settings) + user_creator.create() + + # Retrieve OS creds for new user for creating other OpenStack instance + user_creds = user_creator.get_os_creds(os_creds.project_name) + + # Perform logic + ... + + # Cleanup + user_creator.clean() + +Create Project +-------------- +- Project - snaps.openstack.create\_project.OpenStackProject + + - snaps.openstack.create\_project.ProjectSettings + + - name - the project name (required) + - domain - the project's domain (default='default') + - description - the project's description (optional) + - enables - flag to determine whether or not the project should + be enabled (default=True) + +.. code:: python + + from snaps.openstack.create_project import ProjectSettings, OpenStackProject + project_settings = ProjectSettings(name='username', password='password) + project_creator = OpenStackProject(os_creds, project_settings) + project_creator.create() + + # Perform logic + ... + + # Cleanup + project_creator.clean() + +Create Flavor +------------- +- Flavor - snaps.openstack.create\_flavor.OpenStackFlavor + + - snaps.openstack.create\_flavor.FlavorSettings + + - name - the flavor name (required) + - flavor\_id - the flavor's string ID (default='auto') + - ram - memory in MB to allocate to VM (required) + - disk - disk storage in GB (required) + - vcpus - the number of CPUs to allocate to VM (required) + - ephemeral - the size of the ephemeral disk in GB (default=0) + - swap - the size of the swap disk in GB (default=0) + - rxtx\_factor - the receive/transmit factor to be set on ports + if backend supports QoS extension (default=1.0) + - is\_public - flag that denotes whether or not other projects + can access image (default=True) + +.. code:: python + + from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor + flavor_settings = FlavorSettings(name='flavor-name', ram=4, disk=10, vcpus=2) + flavor_creator = OpenStackFlavor(os_creds, flavor_settings) + flavor_creator.create() + + # Perform logic + ... + + # Cleanup + flavor_creator.clean() + +Create Image +------------ +- Image - snaps.openstack.create\_image.OpenStackImage + + - snaps.openstack.create\_image.ImageSettings + + - name - the image name (required) + - image\_user - the default image user generally used by + OpenStackVMInstance class for obtaining an SSH connection + (required) + - img\_format - the image's format (i.e. qcow2) (required) + - url - the download URL to obtain the image file (this or + image\_file must be configured, not both) + - image\_file - the location of the file to be sourced from the + local filesystem (this or url must be configured, not both) + - nic\_config\_pb\_loc - the location of the ansible playbook + that can configure additional NICs. Floating IPs are required + to perform this operation. (optional) + + +.. code:: python + + from snaps.openstack.create_image import ImageSettings, OpenStackImage + image_settings = ImageSettings(name='image-name', image_user='ubuntu', img_format='qcow2', + url='http://uec-images.ubuntu.com/releases/trusty/14.04/ubuntu-14.04-server-cloudimg-amd64-disk1.img') + image_creator = OpenStackImage(os_creds, image_settings) + image_creator.create() + + # Perform logic + ... + + # Cleanup + image_creator.clean() + +Create Keypair +-------------- +- Keypair - snaps.openstack.create\_keypair.OpenStackKeypair + + - snaps.openstack.create\_keypair.KeypairSettings + + - name - the keypair name (required) + - public\_filepath - the file location to where the public key is + to be written or currently resides (optional) + - private\_filepath - the file location to where the private key + file is to be written or currently resides (optional but highly + recommended to leverage or the private key will be lost + forever) + +.. code:: python + + from snaps.openstack.create_keypair import KeypairSettings, OpenStackKeypair + keypair_settings = KeypairSettings(name='kepair-name', private_filepath='/tmp/priv-kp') + keypair_creator = OpenStackKeypair(os_creds, keypair_settings) + keypair_creator.create() + + # Perform logic + ... + + # Cleanup + keypair_creator.clean() + +Create Network +-------------- + +- Network - snaps.openstack.create\_network.OpenStackNetwork + + - snaps.openstack.create\_network.NetworkSettings + + - name - the name of the network (required) + - admin\_state\_up - flag denoting the administrative status of + the network (True = up, False = down) + - shared - flag indicating whether the network can be shared + across projects/tenants (default=True) + - project\_name - the name of the project (optional - can only be + set by admin users) + - external - flag determining if network has external access + (default=False) + - network\_type - the type of network (i.e. vlan\|vxlan\|flat) + - physical\_network - the name of the physical network (required + when network\_type is 'flat') + - subnet\_settings (list of optional + snaps.openstack.create\_network.SubnetSettings objects) + + - cidr - the subnet's CIDR (required) + - ip\_version - 4 or 6 (default=4) + - name - the subnet name (required) + - project\_name - the name of the project (optional - can only + be set by admin users) + - start - the start address for the allocation pools + - end - the end address for the allocation pools + - gateway\_ip - the gateway IP + - enable\_dhcp - flag to determine whether or not to enable + DHCP (optional) + - dns\_nameservers - a list of DNS nameservers + - host\_routes - list of host route dictionaries for subnet + (optional, see pydoc and Neutron API for more details) + - destination - the destination for static route (optional) + - nexthop - the next hop for the destination (optional) + - ipv6\_ra\_mode - valid values include: 'dhcpv6-stateful', + 'dhcp6v-stateless', 'slaac' (optional) + - ipvc\_address\_mode - valid values include: + 'dhcpv6-stateful', 'dhcp6v-stateless', 'slaac' (optional) + +.. code:: python + + from snaps.openstack.create_network import NetworkSettings, SubnetSettings, OpenStackNetwork + + subnet_settings = SubnetSettings(name='subnet-name', cidr='10.0.0.0/24') + network_settings = NetworkSettings(name='network-name', subnet_settings=[subnet_settings]) + + network_creator = OpenStackNetwork(os_creds, network_settings) + network_creator.create() + + # Perform logic + ... + + # Cleanup + network_creator.clean() + +Create Security Group +--------------------- + +- Security Group - + snaps.openstack.create\_security\_group.OpenStackSecurityGroup + + - snaps.openstack.create\_security\_group.SecurityGroupSettings + + - name - the security group's name (required) + - description - the description (optional) + - project\_name - the name of the project (optional - can only be + set by admin users) + - rule\_settings (list of + optional snaps.openstack.create\_security\_group.SecurityGroupRuleSettings + objects) + + - sec\_grp\_name - the name of the associated security group + (required) + - description - the description (optional) + - direction - enum + snaps.openstack.create\_security\_group.Direction (required) + - remote\_group\_id - the group ID to associate with this rule + - protocol - + enum snaps.openstack.create\_security\_group.Protocol + (optional) + - ethertype - + enum snaps.openstack.create\_security\_group.Ethertype + (optional) + - port\_range\_min - the max port number in the range that is + matched by the security group rule (optional) + - port\_range\_max - the min port number in the range that is + matched by the security group rule (optional) + - sec\_grp\_rule - the rule object to a security group rule + object to associate (note: does not work currently) + - remote\_ip\_prefix - the remote IP prefix to associate with + this metering rule packet (optional) + +.. code:: python + + from snaps.openstack.create_security_group import SecurityGroupSettings, SecurityGroupRuleSettings, Direction, + OpenStackSecurityGroup + + rule_settings = SubnetSettings(name='subnet-name', cidr='10.0.0.0/24') + network_settings = NetworkSettings(name='network-name', subnet_settings=[subnet_settings]) + + sec_grp_name = 'sec-grp-name' + rule_settings = SecurityGroupRuleSettings(name=sec_grp_name, direction=Direction.ingress) + security_group_settings = SecurityGroupSettings(name=sec_grp_name, rule_settings=[rule_settings]) + + security_group_creator = OpenStackSecurityGroup(os_creds, security_group_settings) + security_group_creator.create() + + # Perform logic + ... + + # Cleanup + security_group_creator.clean() + +Create Router +------------- + +- Router - snaps.openstack.create\_router.OpenStackRouter + + - snaps.openstack.create\_router.RouterSettings + + - name - the router name (required) + - project\_name - the name of the project (optional - can only be + set by admin users) + - external\_gateway - the name of the external network (optional) + - admin\_state\_up - flag to denote the administrative status of + the router (default=True) + - external\_fixed\_ips - dictionary containing the IP address + parameters (parameter not tested) + - internal\_subnets - list of subnet names to which this router + will connect (optional) + - port\_settings (list of optional + snaps.openstack.create\_router.PortSettings objects) - creates + custom ports to internal subnets (similar to internal\_subnets + with more control) + + - name + - network\_name + - admin\_state\_up + - project\_name - the name of the project (optional - can only + be set by admin users) + - mac\_address + - ip\_addrs + - fixed\_ips + - security\_groups + - allowed\_address\_pairs + - opt\_value + - opt\_name + - device\_owner + - device\_id + +.. code:: python + + from snaps.openstack.create_router import RouterSettings, OpenStackRouter + + router_settings = RouterSettings(name='router-name', external_gateway='external') + router_creator = OpenStackRouter(os_creds, router_settings) + router_creator.create() + + # Perform logic + ... + + # Cleanup + router_creator.clean() + +Create VM Instance +------------------ + +- VM Instances - snaps.openstack.create\_instance.OpenStackVmInstance + + - snaps.openstack.create\_instance.VmInstanceSettings + + - name - the name of the VM (required) + - flavor - the name of the flavor (required) + - port\_settings - list of + snaps.openstack.create\_network.PortSettings objects where each + denote a NIC (see above in create router section for details) + API does not require, but newer NFVIs now require VMs have at + least one network + - security\_group\_names - a list of security group names to + apply to VM + - floating\_ip\_settings (list of + snaps.openstack\_create\_instance.FloatingIpSettings objects) + + - name - a name to a floating IP for easy lookup  + - port\_name - the name of the VM port on which the floating + IP should be applied (required) + - router\_name - the name of the router to the external + network (required) + - subnet\_name - the name of the subnet on which to attach the + floating IP (optional) + - provisioning - when true, this floating IP will be used for + provisioning which will come into play once we are able to + get multiple floating IPs working. + + - sudo\_user - overrides the image\_settings.image\_user value + when attempting to connect via SSH + - vm\_boot\_timeout - the number of seconds that the thread will + block when querying the VM's status when building (default=900) + - vm\_delete\_timeout - the number of seconds that the thread + will block when querying the VM's status when deleting + (default=300) + - ssh\_connect\_timeout - the number of seconds that the thread + will block when attempting to obtain an SSH connection + (default=180) + - availability\_zone - the name of the compute server on which to + deploy the VM (optional must be admin) + - userdata - the cloud-init script to execute after VM has been + started + + - image\_settings - see snaps.openstack.create\_image.ImageSettings + above (required) + - keypair\_settings - see + snaps.openstack.create\_keypairs.KeypairSettings above (optional) + +.. code:: python + + from snaps.openstack.create_instance import VmInstanceSettings, FloatingIpSettings, OpenStackVmInstance + from snaps.openstack.create_network import PortSettings + + port_settings = PortSettings(name='port-name', network_name=network_settings.name) + floating_ip_settings = FloatingIpSettings(name='fip1', port_name=port_settings.name, router_name=router_settings.name + instance_settings = VmInstanceSettings(name='vm-name', flavor='flavor_settings.name, port_settings=[port_settings], + floating_ip_settings=[floating_ip_settings] + + instance_creator = OpenStackVmInstance(os_creds, instance_settings, image_settings, kepair_settings) + instance_creator.create() + + # Perform logic + ... + ssh_client = instance_creator.ssh_client() + ... + + # Cleanup + instance_creator.clean() + +Ansible Provisioning +==================== + +Being able to easily create OpenStack instances such as virtual networks +and VMs is a good start to the problem of NFV; however, an NFVI is +useless unless there is some software performing some function. This is +why we added Ansible playbook support to SNAPS-OO which can be located +in snaps.provisioning.ansible\_utils#apply\_playbook. See below for a +description of that function's parameters: + +- playbook\_path - the file location of the ansible playbook +- hosts\_inv - a list of hosts/IP addresses to which the playbook will + be applied +- host\_user - the user (preferably sudo) to use for applying the + playbook +- ssh\_priv\_key\_file\_path - the location to the private key file + used for SSH +- variables - a dict() of substitution values for Jinga2 templates + leveraged by Ansible +- proxy\_setting - used to extract the SSH proxy command (optional) + +Apply Ansible Playbook Utility +------------------------------ + +.. code:: python + + from snaps.provisioning import ansible_utils + + ansible_utils.apply_playbook(playbook_path='/tmp/playbook.yml', + hosts_inv=[instance_creator.get_port_ip(instance_settings.port_settings[0].name, + host_user=instance_creator.get_image_user(), + ssh_priv_key_file_path=instance_creator.keypair_settings.private_filepath) + + +OpenStack Utilities +======================================= + +For those who do like working procedurally, SNAPS-OO also leverages +utilitarian functions for nearly every query or request made to +OpenStack. This pattern will make it easier to deal with API version +changes as they would all be made in one place. (see keystone\_utils for +an example of this pattern as this is the only API where SNAPS is +supporting more than one version) + +- snaps.openstack.utils.keystone\_utils - for calls to the Keystone + APIs +- snaps.openstack.utils.glance\_utils - for calls to the Glance APIs +- snaps.openstack.utils.neutron\_utils - for calls to the Neutron APIs +- snaps.openstack.utils.nova\_utils - for calls to the Nova APIs diff --git a/docs/how-to-use/Testing.rst b/docs/how-to-use/Testing.rst index 586974a..7627927 100644 --- a/docs/how-to-use/Testing.rst +++ b/docs/how-to-use/Testing.rst @@ -1,23 +1,14 @@ Running Unit Test Suite ======================= -These tests are written in Python and require an that it is setup before running the tests. -See `install directions `__ for Python installation instructions. - -Start by cloning the snaps-provisioning repository --------------------------------------------------- - -``git clone https://gerrit.cablelabs.com/snaps-provisioning`` - -Install Library ---------------- - -``pip install -e /`` - Execute the tests ----------------- -| ``cd python snaps/unit_test_suite.py -e [path to RC file] -n [external network name]`` +:: + +cd +python snaps/unit_test_suite.py -e -n + | \* All Supported Arguments | \* -e [required - The path to the OpenStack RC file] | \* -n [required - The name of the external network to use for routers @@ -30,15 +21,3 @@ Execute the tests | \* -f [optional - When set, will not execute tests requiring Floating IPS] | \* -u [optional - When set, the unit tests will be executed] - -Test descriptions -================= - -`Unit Testing `__ - Tests that do not require a connection to OpenStack --------------------------------------------------------------------------------------- - -`OpenStack API Tests `__ - Tests many individual OpenStack API calls ----------------------------------------------------------------------------------- - -`Integration Tests `__ - Tests OpenStack object creation in a context. These tests will be run within a custom project as a specific user. ----------------------------------------------------------------------------------------------------------------------------------------------------------------- diff --git a/docs/how-to-use/VirtEnvDeploy.rst b/docs/how-to-use/VirtEnvDeploy.rst index 7f55f0c..58a33d5 100644 --- a/docs/how-to-use/VirtEnvDeploy.rst +++ b/docs/how-to-use/VirtEnvDeploy.rst @@ -1,52 +1,47 @@ -Overview -======== +Try an example +============== -The main purpose of this project is to enable one to describe a virtual environment in a YAML file and enable the -user to deploy it to an OpenStack cloud in a repeatable manner. There are also options to un-deploy that same -environment by leveraging the original YAML file. +Use launcher.py to deploy and clean up example environments. These examples are described in YAML files. -To deploy/clean virtual environments -==================================== +#. Add your OpenStack connection information to the deploy-complex-network.yaml. -- Clone Repository + Edit /examples/complex-network/deploy-complex-network.yaml - - git clone https://gerrit.cablelabs.com/snaps-provisioning + - openstack: the top level tag that denotes configuration for the OpenStack components -- Install Library - - - pip install -e / + - connection: - contains the credentials and endpoints required to + connect with OpenStack + - username: - the project's user (required) + - password: - the tentant's user password (required) + - auth\_url: - the URL to the OpenStack APIs (required) + - project\_name: - the name of the OpenStack project for the user + (required) + - http\_proxy: - the {{ host }}:{{ port }} of the proxy server the + HTTPPhotoman01(optional) -- Deploy +#. Go to the examples directory. - - cd - - python snaps/deploy\_venv.py -e -d - - Working example: + :: -:: + cd /examples/ - python deploy_venv.py -e /examples/complex-network/deploy-complex-network.yaml -d +#. Deploy the launcher. -- Clean + :: - - python deploy\_venv.py -e -c - - Working example (cleanup of a previously deployed virtual - environment where the VM has Yardstick installed): + $ python launcher.py -e ./complex-network/deploy-complex-network.yaml -d -:: +#. Clean the deployment. - python deploy_venv.py -e /examples/complex-network/deploy-complex-network.yaml -c + :: -Environment Configuration YAML File -=================================== + $ python launcher.py -e ./complex-network/deploy-complex-network.yaml -c -The configuration file used to deploy and provision a virtual environment has been designed to describe the required -images, networks, SSH public and private keys, associated VMs, as well as any required post deployment provisioning -tasks. +#. Customize the deployment by changing the yaml file. -\*\*\* Please note that many of the more esoteric optional supported -attributes still have not been fully tested. *** -*** Some of the nested bullets are being hidden by GitLabs, please see -doc/VirtEnvDeploy.md.\*\*\* + The configuration file used to deploy and provision a virtual environment has been designed to describe the required + images, networks, SSH public and private keys, associated VMs, as well as any required post deployment provisioning + tasks. - openstack: the top level tag that denotes configuration for the OpenStack components diff --git a/docs/how-to-use/index.rst b/docs/how-to-use/index.rst index 58b67a3..70f565a 100644 --- a/docs/how-to-use/index.rst +++ b/docs/how-to-use/index.rst @@ -1,26 +1,11 @@ -************************* -Runtime Environment Setup -************************* -- Python 2.7 (recommend leveraging a Virtual Python runtime, e.g. - `Virtualenv `__, in your development - environment) -- Development packages for python and openssl. On CentOS/RHEL: +.. toctree:: + :maxdepth: 2 - # yum install python-devel openssl-devel - -On Ubuntu: - -:: - - # apt-get install python2.7-dev libssl-dev - -- Install SNAPS Library - - - pip install -e /snaps/ - -`Testing `__ -------------------------- - -`Virtual Environment Deployment `__ ------------------------------------------------------- + InstallSnaps + VirtEnvDeploy + Testing + LibraryUsage + APITests + UnitTests + IntegrationTests diff --git a/examples/complex-network/deploy-complex-network.yaml b/examples/complex-network/deploy-complex-network.yaml index 42559e8..eff36c9 100644 --- a/examples/complex-network/deploy-complex-network.yaml +++ b/examples/complex-network/deploy-complex-network.yaml @@ -17,8 +17,8 @@ openstack: connection: # Note - when http_proxy is set, you must also configure ssh for proxy tunneling on your host. username: admin - password: cable123 - auth_url: http://10.197.103.50:5000/v2.0/ + password: NotMyPASS! + auth_url: http://10.10.10.50:5000/v2.0/ project_name: admin http_proxy: localhost:3128 images: diff --git a/examples/external-network/deploy-ext-net.yaml b/examples/external-network/deploy-ext-net.yaml index 31c41ec..ac5e214 100644 --- a/examples/external-network/deploy-ext-net.yaml +++ b/examples/external-network/deploy-ext-net.yaml @@ -17,8 +17,8 @@ openstack: connection: # Note - when http_proxy is set, you must also configure ssh for proxy tunneling on your host. username: admin - password: cable123 - auth_url: http://10.197.103.50:5000/v2.0/ + password: NotMyPAss! + auth_url: http://10.10.10.50:5000/v2.0/ project_name: admin http_proxy: localhost:3128 images: diff --git a/examples/launch.py b/examples/launch.py new file mode 100644 index 0000000..5f2e999 --- /dev/null +++ b/examples/launch.py @@ -0,0 +1,635 @@ +#!/usr/bin/python +# +# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") +# and others. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is responsible for deploying virtual environments +import argparse +import logging +import os +import re + +from snaps import file_utils +from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor +from snaps.openstack.create_image import ImageSettings +from snaps.openstack.create_instance import VmInstanceSettings +from snaps.openstack.create_network import PortSettings, NetworkSettings +from snaps.openstack.create_router import RouterSettings +from snaps.openstack.create_keypairs import KeypairSettings +from snaps.openstack.os_credentials import OSCreds, ProxySettings +from snaps.openstack.utils import deploy_utils +from snaps.provisioning import ansible_utils + +__author__ = 'spisarski' + +logger = logging.getLogger('deploy_venv') + +ARG_NOT_SET = "argument not set" + + +def __get_os_credentials(os_conn_config): + """ + Returns an object containing all of the information required to access OpenStack APIs + :param os_conn_config: The configuration holding the credentials + :return: an OSCreds instance + """ + proxy_settings = None + http_proxy = os_conn_config.get('http_proxy') + if http_proxy: + tokens = re.split(':', http_proxy) + ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd') + proxy_settings = ProxySettings(tokens[0], tokens[1], ssh_proxy_cmd) + + return OSCreds(username=os_conn_config.get('username'), + password=os_conn_config.get('password'), + auth_url=os_conn_config.get('auth_url'), + project_name=os_conn_config.get('project_name'), + proxy_settings=proxy_settings) + + +def __parse_ports_config(config): + """ + Parses the "ports" configuration + :param config: The dictionary to parse + :param os_creds: The OpenStack credentials object + :return: a list of PortConfig objects + """ + out = list() + for port_config in config: + out.append(PortSettings(config=port_config.get('port'))) + return out + + +def __create_flavors(os_conn_config, flavors_config, cleanup=False): + """ + Returns a dictionary of flavors where the key is the image name and the value is the image object + :param os_conn_config: The OpenStack connection credentials + :param flavors_config: The list of image configurations + :param cleanup: Denotes whether or not this is being called for cleanup or not + :return: dictionary + """ + flavors = {} + + if flavors_config: + try: + for flavor_config_dict in flavors_config: + flavor_config = flavor_config_dict.get('flavor') + if flavor_config and flavor_config.get('name'): + flavor_creator = OpenStackFlavor(__get_os_credentials(os_conn_config), + FlavorSettings(flavor_config)) + flavor_creator.create(cleanup=cleanup) + flavors[flavor_config['name']] = flavor_creator + except Exception as e: + for key, flavor_creator in flavors.iteritems(): + flavor_creator.clean() + raise e + logger.info('Created configured flavors') + + return flavors + + +def __create_images(os_conn_config, images_config, cleanup=False): + """ + Returns a dictionary of images where the key is the image name and the value is the image object + :param os_conn_config: The OpenStack connection credentials + :param images_config: The list of image configurations + :param cleanup: Denotes whether or not this is being called for cleanup or not + :return: dictionary + """ + images = {} + + if images_config: + try: + for image_config_dict in images_config: + image_config = image_config_dict.get('image') + if image_config and image_config.get('name'): + images[image_config['name']] = deploy_utils.create_image(__get_os_credentials(os_conn_config), + ImageSettings(image_config), cleanup) + except Exception as e: + for key, image_creator in images.iteritems(): + image_creator.clean() + raise e + logger.info('Created configured images') + + return images + + +def __create_networks(os_conn_config, network_confs, cleanup=False): + """ + Returns a dictionary of networks where the key is the network name and the value is the network object + :param os_conn_config: The OpenStack connection credentials + :param network_confs: The list of network configurations + :param cleanup: Denotes whether or not this is being called for cleanup or not + :return: dictionary + """ + network_dict = {} + + if network_confs: + try: + for network_conf in network_confs: + net_name = network_conf['network']['name'] + os_creds = __get_os_credentials(os_conn_config) + network_dict[net_name] = deploy_utils.create_network( + os_creds, NetworkSettings(config=network_conf['network']), cleanup) + except Exception as e: + for key, net_creator in network_dict.iteritems(): + net_creator.clean() + raise e + + logger.info('Created configured networks') + + return network_dict + + +def __create_routers(os_conn_config, router_confs, cleanup=False): + """ + Returns a dictionary of networks where the key is the network name and the value is the network object + :param os_conn_config: The OpenStack connection credentials + :param router_confs: The list of router configurations + :param cleanup: Denotes whether or not this is being called for cleanup or not + :return: dictionary + """ + router_dict = {} + os_creds = __get_os_credentials(os_conn_config) + + if router_confs: + try: + for router_conf in router_confs: + router_name = router_conf['router']['name'] + router_dict[router_name] = deploy_utils.create_router( + os_creds, RouterSettings(config=router_conf['router']), cleanup) + except Exception as e: + for key, router_creator in router_dict.iteritems(): + router_creator.clean() + raise e + + logger.info('Created configured networks') + + return router_dict + + +def __create_keypairs(os_conn_config, keypair_confs, cleanup=False): + """ + Returns a dictionary of keypairs where the key is the keypair name and the value is the keypair object + :param os_conn_config: The OpenStack connection credentials + :param keypair_confs: The list of keypair configurations + :param cleanup: Denotes whether or not this is being called for cleanup or not + :return: dictionary + """ + keypairs_dict = {} + if keypair_confs: + try: + for keypair_dict in keypair_confs: + keypair_config = keypair_dict['keypair'] + kp_settings = KeypairSettings(keypair_config) + keypairs_dict[keypair_config['name']] = deploy_utils.create_keypair( + __get_os_credentials(os_conn_config), kp_settings, cleanup) + except Exception as e: + for key, keypair_creator in keypairs_dict.iteritems(): + keypair_creator.clean() + raise e + + logger.info('Created configured keypairs') + + return keypairs_dict + + +def __create_instances(os_conn_config, instances_config, image_dict, keypairs_dict, cleanup=False): + """ + Returns a dictionary of instances where the key is the instance name and the value is the VM object + :param os_conn_config: The OpenStack connection credentials + :param instances_config: The list of VM instance configurations + :param image_dict: A dictionary of images that will probably be used to instantiate the VM instance + :param keypairs_dict: A dictionary of keypairs that will probably be used to instantiate the VM instance + :param cleanup: Denotes whether or not this is being called for cleanup or not + :return: dictionary + """ + os_creds = __get_os_credentials(os_conn_config) + + vm_dict = {} + + if instances_config: + try: + for instance_config in instances_config: + conf = instance_config.get('instance') + if conf: + if image_dict: + image_creator = image_dict.get(conf.get('imageName')) + if image_creator: + instance_settings = VmInstanceSettings(config=instance_config['instance']) + kp_name = conf.get('keypair_name') + vm_dict[conf['name']] = deploy_utils.create_vm_instance( + os_creds, instance_settings, image_creator.image_settings, + keypair_creator=keypairs_dict[kp_name], cleanup=cleanup) + else: + raise Exception('Image creator instance not found. Cannot instantiate') + else: + raise Exception('Image dictionary is None. Cannot instantiate') + else: + raise Exception('Instance configuration is None. Cannot instantiate') + except Exception as e: + logger.error('Unexpected error creating instances. Attempting to cleanup environment - ' + e.message) + for key, inst_creator in vm_dict.iteritems(): + inst_creator.clean() + raise e + + logger.info('Created configured instances') + + return vm_dict + + +def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict, image_dict, flavor_dict, env_file): + """ + Applies ansible playbooks to running VMs with floating IPs + :param ansible_configs: a list of Ansible configurations + :param os_conn_config: the OpenStack connection configuration used to create an OSCreds instance + :param vm_dict: the dictionary of newly instantiated VMs where the name is the key + :param image_dict: the dictionary of newly instantiated images where the name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key + :param env_file: the path of the environment for setting the CWD so playbook location is relative to the deployment + file + :return: t/f - true if successful + """ + logger.info("Applying Ansible Playbooks") + if ansible_configs: + # Ensure all hosts are accepting SSH session requests + for vm_inst in vm_dict.values(): + if not vm_inst.vm_ssh_active(block=True): + logger.warn("Timeout waiting for instance to respond to SSH requests") + return False + + # Set CWD so the deployment file's playbook location can leverage relative paths + orig_cwd = os.getcwd() + env_dir = os.path.dirname(env_file) + os.chdir(env_dir) + + # Apply playbooks + for ansible_config in ansible_configs: + os_creds = __get_os_credentials(os_conn_config) + __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict) + + # Return to original directory + os.chdir(orig_cwd) + + return True + + +def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict): + """ + Applies an Ansible configuration setting + :param ansible_config: the configuration settings + :param os_creds: the OpenStack credentials object + :param vm_dict: the dictionary of newly instantiated VMs where the name is the key + :param image_dict: the dictionary of newly instantiated images where the name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key + """ + if ansible_config: + remote_user, floating_ips, private_key_filepath, proxy_settings = __get_connection_info(ansible_config, vm_dict) + if floating_ips: + retval = ansible_utils.apply_playbook( + ansible_config['playbook_location'], floating_ips, remote_user, private_key_filepath, + variables=__get_variables(ansible_config.get('variables'), os_creds, vm_dict, image_dict, flavor_dict), + proxy_setting=proxy_settings) + if retval != 0: + # Not a fatal type of event + logger.warn('Unable to apply playbook found at location - ' + ansible_config('playbook_location')) + + +def __get_connection_info(ansible_config, vm_dict): + """ + Returns a tuple of data required for connecting to the running VMs + (remote_user, [floating_ips], private_key_filepath, proxy_settings) + :param ansible_config: the configuration settings + :param vm_dict: the dictionary of VMs where the VM name is the key + :return: tuple where the first element is the user and the second is a list of floating IPs and the third is the + private key file location and the fourth is an instance of the snaps.ProxySettings class + (note: in order to work, each of the hosts need to have the same sudo_user and private key file location values) + """ + if ansible_config.get('hosts'): + hosts = ansible_config['hosts'] + if len(hosts) > 0: + floating_ips = list() + remote_user = None + private_key_filepath = None + proxy_settings = None + for host in hosts: + vm = vm_dict.get(host) + if vm: + fip = vm.get_floating_ip() + if fip: + remote_user = vm.get_image_user() + + if fip: + floating_ips.append(fip.ip) + else: + raise Exception('Could not find floating IP for VM - ' + vm.name) + + private_key_filepath = vm.keypair_settings.private_filepath + proxy_settings = vm.get_os_creds().proxy_settings + else: + logger.error('Could not locate VM with name - ' + host) + + return remote_user, floating_ips, private_key_filepath, proxy_settings + return None + + +def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict): + """ + Returns a dictionary of substitution variables to be used for Ansible templates + :param var_config: the variable configuration settings + :param os_creds: the OpenStack credentials object + :param vm_dict: the dictionary of newly instantiated VMs where the name is the key + :param image_dict: the dictionary of newly instantiated images where the name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key + :return: dictionary or None + """ + if var_config and vm_dict and len(vm_dict) > 0: + variables = dict() + for key, value in var_config.iteritems(): + value = __get_variable_value(value, os_creds, vm_dict, image_dict, flavor_dict) + if key and value: + variables[key] = value + logger.info("Set Jinga2 variable with key [" + key + "] the value [" + value + ']') + else: + logger.warn('Key [' + str(key) + '] or Value [' + str(value) + '] must not be None') + return variables + return None + + +def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict, flavor_dict): + """ + Returns the associated variable value for use by Ansible for substitution purposes + :param var_config_values: the configuration dictionary + :param os_creds: the OpenStack credentials object + :param vm_dict: the dictionary of newly instantiated VMs where the name is the key + :param image_dict: the dictionary of newly instantiated images where the name is the key + :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key + :return: + """ + if var_config_values['type'] == 'string': + return __get_string_variable_value(var_config_values) + if var_config_values['type'] == 'vm-attr': + return __get_vm_attr_variable_value(var_config_values, vm_dict) + if var_config_values['type'] == 'os_creds': + return __get_os_creds_variable_value(var_config_values, os_creds) + if var_config_values['type'] == 'port': + return __get_vm_port_variable_value(var_config_values, vm_dict) + if var_config_values['type'] == 'image': + return __get_image_variable_value(var_config_values, image_dict) + if var_config_values['type'] == 'flavor': + return __get_flavor_variable_value(var_config_values, flavor_dict) + return None + + +def __get_string_variable_value(var_config_values): + """ + Returns the associated string value + :param var_config_values: the configuration dictionary + :return: the value contained in the dictionary with the key 'value' + """ + return var_config_values['value'] + + +def __get_vm_attr_variable_value(var_config_values, vm_dict): + """ + Returns the associated value contained on a VM instance + :param var_config_values: the configuration dictionary + :param vm_dict: the dictionary containing all VMs where the key is the VM's name + :return: the value + """ + vm = vm_dict.get(var_config_values['vm_name']) + if vm: + if var_config_values['value'] == 'floating_ip': + return vm.get_floating_ip().ip + if var_config_values['value'] == 'image_user': + return vm.get_image_user() + + +def __get_os_creds_variable_value(var_config_values, os_creds): + """ + Returns the associated OS credentials value + :param var_config_values: the configuration dictionary + :param os_creds: the credentials + :return: the value + """ + logger.info("Retrieving OS Credentials") + if os_creds: + if var_config_values['value'] == 'username': + logger.info("Returning OS username") + return os_creds.username + elif var_config_values['value'] == 'password': + logger.info("Returning OS password") + return os_creds.password + elif var_config_values['value'] == 'auth_url': + logger.info("Returning OS auth_url") + return os_creds.auth_url + elif var_config_values['value'] == 'project_name': + logger.info("Returning OS project_name") + return os_creds.project_name + + logger.info("Returning none") + return None + + +def __get_vm_port_variable_value(var_config_values, vm_dict): + """ + Returns the associated OS credentials value + :param var_config_values: the configuration dictionary + :param vm_dict: the dictionary containing all VMs where the key is the VM's name + :return: the value + """ + port_name = var_config_values.get('port_name') + vm_name = var_config_values.get('vm_name') + + if port_name and vm_name: + vm = vm_dict.get(vm_name) + if vm: + port_value_id = var_config_values.get('port_value') + if port_value_id: + if port_value_id == 'mac_address': + return vm.get_port_mac(port_name) + if port_value_id == 'ip_address': + return vm.get_port_ip(port_name) + + +def __get_image_variable_value(var_config_values, image_dict): + """ + Returns the associated image value + :param var_config_values: the configuration dictionary + :param image_dict: the dictionary containing all images where the key is the name + :return: the value + """ + logger.info("Retrieving image values") + + if image_dict: + if var_config_values.get('image_name'): + image_creator = image_dict.get(var_config_values['image_name']) + if image_creator: + if var_config_values.get('value') and var_config_values['value'] == 'id': + return image_creator.get_image().id + if var_config_values.get('value') and var_config_values['value'] == 'user': + return image_creator.image_settings.image_user + + logger.info("Returning none") + return None + + +def __get_flavor_variable_value(var_config_values, flavor_dict): + """ + Returns the associated flavor value + :param var_config_values: the configuration dictionary + :param flavor_dict: the dictionary containing all flavor creators where the key is the name + :return: the value or None + """ + logger.info("Retrieving flavor values") + + if flavor_dict: + if var_config_values.get('flavor_name'): + flavor_creator = flavor_dict.get(var_config_values['flavor_name']) + if flavor_creator: + if var_config_values.get('value') and var_config_values['value'] == 'id': + return flavor_creator.get_flavor().id + + logger.info("Returning none") + return None + + +def main(arguments): + """ + Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or ... + Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg containing the following content: + + [defaults] + host_key_checking = False + + CWD must be this directory where this script is located. + + :return: To the OS + """ + log_level = logging.INFO + if arguments.log_level != 'INFO': + log_level = logging.DEBUG + logging.basicConfig(level=log_level) + + logger.info('Starting to Deploy') + config = file_utils.read_yaml(arguments.environment) + logger.info('Read configuration file - ' + arguments.environment) + + if config: + os_config = config.get('openstack') + + image_dict = {} + network_dict = {} + router_dict = {} + keypairs_dict = {} + vm_dict = {} + + if os_config: + try: + os_conn_config = os_config.get('connection') + + # Create flavors + flavor_dict = __create_flavors(os_conn_config, os_config.get('flavors'), + arguments.clean is not ARG_NOT_SET) + + # Create images + image_dict = __create_images(os_conn_config, os_config.get('images'), + arguments.clean is not ARG_NOT_SET) + + # Create network + network_dict = __create_networks(os_conn_config, os_config.get('networks'), + arguments.clean is not ARG_NOT_SET) + + # Create network + router_dict = __create_routers(os_conn_config, os_config.get('routers'), + arguments.clean is not ARG_NOT_SET) + + # Create keypairs + keypairs_dict = __create_keypairs(os_conn_config, os_config.get('keypairs'), + arguments.clean is not ARG_NOT_SET) + + # Create instance + vm_dict = __create_instances(os_conn_config, os_config.get('instances'), image_dict, keypairs_dict, + arguments.clean is not ARG_NOT_SET) + logger.info('Completed creating/retrieving all configured instances') + except Exception as e: + logger.error('Unexpected error deploying environment. Rolling back due to - ' + e.message) + __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, True) + raise e + + + # Must enter either block + if arguments.clean is not ARG_NOT_SET: + # Cleanup Environment + __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, + arguments.clean_image is not ARG_NOT_SET) + elif arguments.deploy is not ARG_NOT_SET: + logger.info('Configuring NICs where required') + for vm in vm_dict.itervalues(): + vm.config_nics() + logger.info('Completed NIC configuration') + + # Provision VMs + ansible_config = config.get('ansible') + if ansible_config and vm_dict: + if not __apply_ansible_playbooks(ansible_config, os_conn_config, vm_dict, image_dict, flavor_dict, + arguments.environment): + logger.error("Problem applying ansible playbooks") + else: + logger.error('Unable to read configuration file - ' + arguments.environment) + exit(1) + + exit(0) + + +def __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, clean_image=False): + for key, vm_inst in vm_dict.iteritems(): + vm_inst.clean() + for key, kp_inst in keypairs_dict.iteritems(): + kp_inst.clean() + for key, router_inst in router_dict.iteritems(): + router_inst.clean() + for key, net_inst in network_dict.iteritems(): + net_inst.clean() + if clean_image: + for key, image_inst in image_dict.iteritems(): + image_inst.clean() + for key, flavor_inst in flavor_dict.iteritems(): + flavor_inst.clean() + + +if __name__ == '__main__': + # To ensure any files referenced via a relative path will begin from the diectory in which this file resides + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET, + help='When used, environment will be deployed and provisioned') + parser.add_argument('-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET, + help='When used, the environment will be removed') + parser.add_argument('-i', '--clean-image', dest='clean_image', nargs='?', default=ARG_NOT_SET, + help='When cleaning, if this is set, the image will be cleaned too') + parser.add_argument('-e', '--env', dest='environment', required=True, + help='The environment configuration YAML file - REQUIRED') + parser.add_argument('-l', '--log-level', dest='log_level', default='INFO', help='Logging Level (INFO|DEBUG)') + args = parser.parse_args() + + if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET: + print 'Must enter either -d for deploy or -c for cleaning up and environment' + exit(1) + if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET: + print 'Cannot enter both options -d/--deploy and -c/--clean' + exit(1) + main(args) diff --git a/examples/simple/deploy-simple.yaml b/examples/simple/deploy-simple.yaml index ae946de..982a676 100644 --- a/examples/simple/deploy-simple.yaml +++ b/examples/simple/deploy-simple.yaml @@ -17,8 +17,7 @@ openstack: connection: # Note - when http_proxy is set, you must also configure ssh for proxy tunneling on your host. username: admin - password: cable123 -# auth_url: http://10.197.103.50:5000/v2.0/ + password: NotMyPASS! auth_url: http://192.168.67.10:5000/v2.0 project_name: admin http_proxy: 10.197.123.27:3128 diff --git a/examples/two-network/deploy-two-net-centos.yaml b/examples/two-network/deploy-two-net-centos.yaml index 4fae4aa..4b5ba89 100644 --- a/examples/two-network/deploy-two-net-centos.yaml +++ b/examples/two-network/deploy-two-net-centos.yaml @@ -17,8 +17,8 @@ openstack: connection: # Note - when http_proxy is set, you must also configure ssh for proxy tunneling on your host. username: admin - password: cable123 - auth_url: http://10.197.103.50:5000/v2.0/ + password: NotMyPass! + auth_url: http://10.10.10.50:5000/v2.0/ project_name: admin http_proxy: localhost:3128 images: diff --git a/examples/two-network/deploy-two-net-ubuntu.yaml b/examples/two-network/deploy-two-net-ubuntu.yaml index ffcb05d..0ad471c 100644 --- a/examples/two-network/deploy-two-net-ubuntu.yaml +++ b/examples/two-network/deploy-two-net-ubuntu.yaml @@ -17,8 +17,8 @@ openstack: connection: # Note - when http_proxy is set, you must also configure ssh for proxy tunneling on your host. username: admin - password: cable123 - auth_url: http://10.197.103.50:5000/v2.0/ + password: NotMyPass! + auth_url: http://10.10.10.50:5000/v2.0/ project_name: admin http_proxy: localhost:3128 images: diff --git a/snaps/deploy_venv.py b/snaps/deploy_venv.py deleted file mode 100644 index 5f2e999..0000000 --- a/snaps/deploy_venv.py +++ /dev/null @@ -1,635 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs") -# and others. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script is responsible for deploying virtual environments -import argparse -import logging -import os -import re - -from snaps import file_utils -from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor -from snaps.openstack.create_image import ImageSettings -from snaps.openstack.create_instance import VmInstanceSettings -from snaps.openstack.create_network import PortSettings, NetworkSettings -from snaps.openstack.create_router import RouterSettings -from snaps.openstack.create_keypairs import KeypairSettings -from snaps.openstack.os_credentials import OSCreds, ProxySettings -from snaps.openstack.utils import deploy_utils -from snaps.provisioning import ansible_utils - -__author__ = 'spisarski' - -logger = logging.getLogger('deploy_venv') - -ARG_NOT_SET = "argument not set" - - -def __get_os_credentials(os_conn_config): - """ - Returns an object containing all of the information required to access OpenStack APIs - :param os_conn_config: The configuration holding the credentials - :return: an OSCreds instance - """ - proxy_settings = None - http_proxy = os_conn_config.get('http_proxy') - if http_proxy: - tokens = re.split(':', http_proxy) - ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd') - proxy_settings = ProxySettings(tokens[0], tokens[1], ssh_proxy_cmd) - - return OSCreds(username=os_conn_config.get('username'), - password=os_conn_config.get('password'), - auth_url=os_conn_config.get('auth_url'), - project_name=os_conn_config.get('project_name'), - proxy_settings=proxy_settings) - - -def __parse_ports_config(config): - """ - Parses the "ports" configuration - :param config: The dictionary to parse - :param os_creds: The OpenStack credentials object - :return: a list of PortConfig objects - """ - out = list() - for port_config in config: - out.append(PortSettings(config=port_config.get('port'))) - return out - - -def __create_flavors(os_conn_config, flavors_config, cleanup=False): - """ - Returns a dictionary of flavors where the key is the image name and the value is the image object - :param os_conn_config: The OpenStack connection credentials - :param flavors_config: The list of image configurations - :param cleanup: Denotes whether or not this is being called for cleanup or not - :return: dictionary - """ - flavors = {} - - if flavors_config: - try: - for flavor_config_dict in flavors_config: - flavor_config = flavor_config_dict.get('flavor') - if flavor_config and flavor_config.get('name'): - flavor_creator = OpenStackFlavor(__get_os_credentials(os_conn_config), - FlavorSettings(flavor_config)) - flavor_creator.create(cleanup=cleanup) - flavors[flavor_config['name']] = flavor_creator - except Exception as e: - for key, flavor_creator in flavors.iteritems(): - flavor_creator.clean() - raise e - logger.info('Created configured flavors') - - return flavors - - -def __create_images(os_conn_config, images_config, cleanup=False): - """ - Returns a dictionary of images where the key is the image name and the value is the image object - :param os_conn_config: The OpenStack connection credentials - :param images_config: The list of image configurations - :param cleanup: Denotes whether or not this is being called for cleanup or not - :return: dictionary - """ - images = {} - - if images_config: - try: - for image_config_dict in images_config: - image_config = image_config_dict.get('image') - if image_config and image_config.get('name'): - images[image_config['name']] = deploy_utils.create_image(__get_os_credentials(os_conn_config), - ImageSettings(image_config), cleanup) - except Exception as e: - for key, image_creator in images.iteritems(): - image_creator.clean() - raise e - logger.info('Created configured images') - - return images - - -def __create_networks(os_conn_config, network_confs, cleanup=False): - """ - Returns a dictionary of networks where the key is the network name and the value is the network object - :param os_conn_config: The OpenStack connection credentials - :param network_confs: The list of network configurations - :param cleanup: Denotes whether or not this is being called for cleanup or not - :return: dictionary - """ - network_dict = {} - - if network_confs: - try: - for network_conf in network_confs: - net_name = network_conf['network']['name'] - os_creds = __get_os_credentials(os_conn_config) - network_dict[net_name] = deploy_utils.create_network( - os_creds, NetworkSettings(config=network_conf['network']), cleanup) - except Exception as e: - for key, net_creator in network_dict.iteritems(): - net_creator.clean() - raise e - - logger.info('Created configured networks') - - return network_dict - - -def __create_routers(os_conn_config, router_confs, cleanup=False): - """ - Returns a dictionary of networks where the key is the network name and the value is the network object - :param os_conn_config: The OpenStack connection credentials - :param router_confs: The list of router configurations - :param cleanup: Denotes whether or not this is being called for cleanup or not - :return: dictionary - """ - router_dict = {} - os_creds = __get_os_credentials(os_conn_config) - - if router_confs: - try: - for router_conf in router_confs: - router_name = router_conf['router']['name'] - router_dict[router_name] = deploy_utils.create_router( - os_creds, RouterSettings(config=router_conf['router']), cleanup) - except Exception as e: - for key, router_creator in router_dict.iteritems(): - router_creator.clean() - raise e - - logger.info('Created configured networks') - - return router_dict - - -def __create_keypairs(os_conn_config, keypair_confs, cleanup=False): - """ - Returns a dictionary of keypairs where the key is the keypair name and the value is the keypair object - :param os_conn_config: The OpenStack connection credentials - :param keypair_confs: The list of keypair configurations - :param cleanup: Denotes whether or not this is being called for cleanup or not - :return: dictionary - """ - keypairs_dict = {} - if keypair_confs: - try: - for keypair_dict in keypair_confs: - keypair_config = keypair_dict['keypair'] - kp_settings = KeypairSettings(keypair_config) - keypairs_dict[keypair_config['name']] = deploy_utils.create_keypair( - __get_os_credentials(os_conn_config), kp_settings, cleanup) - except Exception as e: - for key, keypair_creator in keypairs_dict.iteritems(): - keypair_creator.clean() - raise e - - logger.info('Created configured keypairs') - - return keypairs_dict - - -def __create_instances(os_conn_config, instances_config, image_dict, keypairs_dict, cleanup=False): - """ - Returns a dictionary of instances where the key is the instance name and the value is the VM object - :param os_conn_config: The OpenStack connection credentials - :param instances_config: The list of VM instance configurations - :param image_dict: A dictionary of images that will probably be used to instantiate the VM instance - :param keypairs_dict: A dictionary of keypairs that will probably be used to instantiate the VM instance - :param cleanup: Denotes whether or not this is being called for cleanup or not - :return: dictionary - """ - os_creds = __get_os_credentials(os_conn_config) - - vm_dict = {} - - if instances_config: - try: - for instance_config in instances_config: - conf = instance_config.get('instance') - if conf: - if image_dict: - image_creator = image_dict.get(conf.get('imageName')) - if image_creator: - instance_settings = VmInstanceSettings(config=instance_config['instance']) - kp_name = conf.get('keypair_name') - vm_dict[conf['name']] = deploy_utils.create_vm_instance( - os_creds, instance_settings, image_creator.image_settings, - keypair_creator=keypairs_dict[kp_name], cleanup=cleanup) - else: - raise Exception('Image creator instance not found. Cannot instantiate') - else: - raise Exception('Image dictionary is None. Cannot instantiate') - else: - raise Exception('Instance configuration is None. Cannot instantiate') - except Exception as e: - logger.error('Unexpected error creating instances. Attempting to cleanup environment - ' + e.message) - for key, inst_creator in vm_dict.iteritems(): - inst_creator.clean() - raise e - - logger.info('Created configured instances') - - return vm_dict - - -def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict, image_dict, flavor_dict, env_file): - """ - Applies ansible playbooks to running VMs with floating IPs - :param ansible_configs: a list of Ansible configurations - :param os_conn_config: the OpenStack connection configuration used to create an OSCreds instance - :param vm_dict: the dictionary of newly instantiated VMs where the name is the key - :param image_dict: the dictionary of newly instantiated images where the name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key - :param env_file: the path of the environment for setting the CWD so playbook location is relative to the deployment - file - :return: t/f - true if successful - """ - logger.info("Applying Ansible Playbooks") - if ansible_configs: - # Ensure all hosts are accepting SSH session requests - for vm_inst in vm_dict.values(): - if not vm_inst.vm_ssh_active(block=True): - logger.warn("Timeout waiting for instance to respond to SSH requests") - return False - - # Set CWD so the deployment file's playbook location can leverage relative paths - orig_cwd = os.getcwd() - env_dir = os.path.dirname(env_file) - os.chdir(env_dir) - - # Apply playbooks - for ansible_config in ansible_configs: - os_creds = __get_os_credentials(os_conn_config) - __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict) - - # Return to original directory - os.chdir(orig_cwd) - - return True - - -def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict): - """ - Applies an Ansible configuration setting - :param ansible_config: the configuration settings - :param os_creds: the OpenStack credentials object - :param vm_dict: the dictionary of newly instantiated VMs where the name is the key - :param image_dict: the dictionary of newly instantiated images where the name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key - """ - if ansible_config: - remote_user, floating_ips, private_key_filepath, proxy_settings = __get_connection_info(ansible_config, vm_dict) - if floating_ips: - retval = ansible_utils.apply_playbook( - ansible_config['playbook_location'], floating_ips, remote_user, private_key_filepath, - variables=__get_variables(ansible_config.get('variables'), os_creds, vm_dict, image_dict, flavor_dict), - proxy_setting=proxy_settings) - if retval != 0: - # Not a fatal type of event - logger.warn('Unable to apply playbook found at location - ' + ansible_config('playbook_location')) - - -def __get_connection_info(ansible_config, vm_dict): - """ - Returns a tuple of data required for connecting to the running VMs - (remote_user, [floating_ips], private_key_filepath, proxy_settings) - :param ansible_config: the configuration settings - :param vm_dict: the dictionary of VMs where the VM name is the key - :return: tuple where the first element is the user and the second is a list of floating IPs and the third is the - private key file location and the fourth is an instance of the snaps.ProxySettings class - (note: in order to work, each of the hosts need to have the same sudo_user and private key file location values) - """ - if ansible_config.get('hosts'): - hosts = ansible_config['hosts'] - if len(hosts) > 0: - floating_ips = list() - remote_user = None - private_key_filepath = None - proxy_settings = None - for host in hosts: - vm = vm_dict.get(host) - if vm: - fip = vm.get_floating_ip() - if fip: - remote_user = vm.get_image_user() - - if fip: - floating_ips.append(fip.ip) - else: - raise Exception('Could not find floating IP for VM - ' + vm.name) - - private_key_filepath = vm.keypair_settings.private_filepath - proxy_settings = vm.get_os_creds().proxy_settings - else: - logger.error('Could not locate VM with name - ' + host) - - return remote_user, floating_ips, private_key_filepath, proxy_settings - return None - - -def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict): - """ - Returns a dictionary of substitution variables to be used for Ansible templates - :param var_config: the variable configuration settings - :param os_creds: the OpenStack credentials object - :param vm_dict: the dictionary of newly instantiated VMs where the name is the key - :param image_dict: the dictionary of newly instantiated images where the name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key - :return: dictionary or None - """ - if var_config and vm_dict and len(vm_dict) > 0: - variables = dict() - for key, value in var_config.iteritems(): - value = __get_variable_value(value, os_creds, vm_dict, image_dict, flavor_dict) - if key and value: - variables[key] = value - logger.info("Set Jinga2 variable with key [" + key + "] the value [" + value + ']') - else: - logger.warn('Key [' + str(key) + '] or Value [' + str(value) + '] must not be None') - return variables - return None - - -def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict, flavor_dict): - """ - Returns the associated variable value for use by Ansible for substitution purposes - :param var_config_values: the configuration dictionary - :param os_creds: the OpenStack credentials object - :param vm_dict: the dictionary of newly instantiated VMs where the name is the key - :param image_dict: the dictionary of newly instantiated images where the name is the key - :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key - :return: - """ - if var_config_values['type'] == 'string': - return __get_string_variable_value(var_config_values) - if var_config_values['type'] == 'vm-attr': - return __get_vm_attr_variable_value(var_config_values, vm_dict) - if var_config_values['type'] == 'os_creds': - return __get_os_creds_variable_value(var_config_values, os_creds) - if var_config_values['type'] == 'port': - return __get_vm_port_variable_value(var_config_values, vm_dict) - if var_config_values['type'] == 'image': - return __get_image_variable_value(var_config_values, image_dict) - if var_config_values['type'] == 'flavor': - return __get_flavor_variable_value(var_config_values, flavor_dict) - return None - - -def __get_string_variable_value(var_config_values): - """ - Returns the associated string value - :param var_config_values: the configuration dictionary - :return: the value contained in the dictionary with the key 'value' - """ - return var_config_values['value'] - - -def __get_vm_attr_variable_value(var_config_values, vm_dict): - """ - Returns the associated value contained on a VM instance - :param var_config_values: the configuration dictionary - :param vm_dict: the dictionary containing all VMs where the key is the VM's name - :return: the value - """ - vm = vm_dict.get(var_config_values['vm_name']) - if vm: - if var_config_values['value'] == 'floating_ip': - return vm.get_floating_ip().ip - if var_config_values['value'] == 'image_user': - return vm.get_image_user() - - -def __get_os_creds_variable_value(var_config_values, os_creds): - """ - Returns the associated OS credentials value - :param var_config_values: the configuration dictionary - :param os_creds: the credentials - :return: the value - """ - logger.info("Retrieving OS Credentials") - if os_creds: - if var_config_values['value'] == 'username': - logger.info("Returning OS username") - return os_creds.username - elif var_config_values['value'] == 'password': - logger.info("Returning OS password") - return os_creds.password - elif var_config_values['value'] == 'auth_url': - logger.info("Returning OS auth_url") - return os_creds.auth_url - elif var_config_values['value'] == 'project_name': - logger.info("Returning OS project_name") - return os_creds.project_name - - logger.info("Returning none") - return None - - -def __get_vm_port_variable_value(var_config_values, vm_dict): - """ - Returns the associated OS credentials value - :param var_config_values: the configuration dictionary - :param vm_dict: the dictionary containing all VMs where the key is the VM's name - :return: the value - """ - port_name = var_config_values.get('port_name') - vm_name = var_config_values.get('vm_name') - - if port_name and vm_name: - vm = vm_dict.get(vm_name) - if vm: - port_value_id = var_config_values.get('port_value') - if port_value_id: - if port_value_id == 'mac_address': - return vm.get_port_mac(port_name) - if port_value_id == 'ip_address': - return vm.get_port_ip(port_name) - - -def __get_image_variable_value(var_config_values, image_dict): - """ - Returns the associated image value - :param var_config_values: the configuration dictionary - :param image_dict: the dictionary containing all images where the key is the name - :return: the value - """ - logger.info("Retrieving image values") - - if image_dict: - if var_config_values.get('image_name'): - image_creator = image_dict.get(var_config_values['image_name']) - if image_creator: - if var_config_values.get('value') and var_config_values['value'] == 'id': - return image_creator.get_image().id - if var_config_values.get('value') and var_config_values['value'] == 'user': - return image_creator.image_settings.image_user - - logger.info("Returning none") - return None - - -def __get_flavor_variable_value(var_config_values, flavor_dict): - """ - Returns the associated flavor value - :param var_config_values: the configuration dictionary - :param flavor_dict: the dictionary containing all flavor creators where the key is the name - :return: the value or None - """ - logger.info("Retrieving flavor values") - - if flavor_dict: - if var_config_values.get('flavor_name'): - flavor_creator = flavor_dict.get(var_config_values['flavor_name']) - if flavor_creator: - if var_config_values.get('value') and var_config_values['value'] == 'id': - return flavor_creator.get_flavor().id - - logger.info("Returning none") - return None - - -def main(arguments): - """ - Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or ... - Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg containing the following content: - - [defaults] - host_key_checking = False - - CWD must be this directory where this script is located. - - :return: To the OS - """ - log_level = logging.INFO - if arguments.log_level != 'INFO': - log_level = logging.DEBUG - logging.basicConfig(level=log_level) - - logger.info('Starting to Deploy') - config = file_utils.read_yaml(arguments.environment) - logger.info('Read configuration file - ' + arguments.environment) - - if config: - os_config = config.get('openstack') - - image_dict = {} - network_dict = {} - router_dict = {} - keypairs_dict = {} - vm_dict = {} - - if os_config: - try: - os_conn_config = os_config.get('connection') - - # Create flavors - flavor_dict = __create_flavors(os_conn_config, os_config.get('flavors'), - arguments.clean is not ARG_NOT_SET) - - # Create images - image_dict = __create_images(os_conn_config, os_config.get('images'), - arguments.clean is not ARG_NOT_SET) - - # Create network - network_dict = __create_networks(os_conn_config, os_config.get('networks'), - arguments.clean is not ARG_NOT_SET) - - # Create network - router_dict = __create_routers(os_conn_config, os_config.get('routers'), - arguments.clean is not ARG_NOT_SET) - - # Create keypairs - keypairs_dict = __create_keypairs(os_conn_config, os_config.get('keypairs'), - arguments.clean is not ARG_NOT_SET) - - # Create instance - vm_dict = __create_instances(os_conn_config, os_config.get('instances'), image_dict, keypairs_dict, - arguments.clean is not ARG_NOT_SET) - logger.info('Completed creating/retrieving all configured instances') - except Exception as e: - logger.error('Unexpected error deploying environment. Rolling back due to - ' + e.message) - __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, True) - raise e - - - # Must enter either block - if arguments.clean is not ARG_NOT_SET: - # Cleanup Environment - __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, - arguments.clean_image is not ARG_NOT_SET) - elif arguments.deploy is not ARG_NOT_SET: - logger.info('Configuring NICs where required') - for vm in vm_dict.itervalues(): - vm.config_nics() - logger.info('Completed NIC configuration') - - # Provision VMs - ansible_config = config.get('ansible') - if ansible_config and vm_dict: - if not __apply_ansible_playbooks(ansible_config, os_conn_config, vm_dict, image_dict, flavor_dict, - arguments.environment): - logger.error("Problem applying ansible playbooks") - else: - logger.error('Unable to read configuration file - ' + arguments.environment) - exit(1) - - exit(0) - - -def __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, clean_image=False): - for key, vm_inst in vm_dict.iteritems(): - vm_inst.clean() - for key, kp_inst in keypairs_dict.iteritems(): - kp_inst.clean() - for key, router_inst in router_dict.iteritems(): - router_inst.clean() - for key, net_inst in network_dict.iteritems(): - net_inst.clean() - if clean_image: - for key, image_inst in image_dict.iteritems(): - image_inst.clean() - for key, flavor_inst in flavor_dict.iteritems(): - flavor_inst.clean() - - -if __name__ == '__main__': - # To ensure any files referenced via a relative path will begin from the diectory in which this file resides - os.chdir(os.path.dirname(os.path.realpath(__file__))) - - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET, - help='When used, environment will be deployed and provisioned') - parser.add_argument('-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET, - help='When used, the environment will be removed') - parser.add_argument('-i', '--clean-image', dest='clean_image', nargs='?', default=ARG_NOT_SET, - help='When cleaning, if this is set, the image will be cleaned too') - parser.add_argument('-e', '--env', dest='environment', required=True, - help='The environment configuration YAML file - REQUIRED') - parser.add_argument('-l', '--log-level', dest='log_level', default='INFO', help='Logging Level (INFO|DEBUG)') - args = parser.parse_args() - - if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET: - print 'Must enter either -d for deploy or -c for cleaning up and environment' - exit(1) - if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET: - print 'Cannot enter both options -d/--deploy and -c/--clean' - exit(1) - main(args) -- cgit 1.2.3-korg