Initial patch with all code from CableLabs repository.

Change-Id: I70a2778718c5e7f21fd14e4ad28c9269d3761cc7
Signed-off-by: spisarski <s.pisarski@cablelabs.com>

12 files changed, 475 insertions, 0 deletions

diff --git a/snaps/provisioning/__init__.py b/snaps/provisioning/__init__.py
new file mode 100644
index 0000000..7f92908
--- /dev/null
+++ b/snaps/provisioning/__init__.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+__author__ = 'spisarski'
\ No newline at end of file
diff --git a/snaps/provisioning/ansible/centos-network-setup/playbooks/configure_host.yml b/snaps/provisioning/ansible/centos-network-setup/playbooks/configure_host.yml
new file mode 100644
index 0000000..8df03cb
--- /dev/null
+++ b/snaps/provisioning/ansible/centos-network-setup/playbooks/configure_host.yml
@@ -0,0 +1,26 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Configure NIC
+  hosts: all
+  become: yes
+  become_method: sudo
+  become_user: root
+
+  tasks:
+   - name: Setup /etc/sysconfig/network-scripts/ifcfg-eth1 file
+     action: template owner=root group=root mode=644 src=../templates/ifcfg-interface dest=/etc/sysconfig/network-scripts/ifcfg-{{nic_name}}
+   - name : Restart Network
+     command: systemctl restart network
\ No newline at end of file
diff --git a/snaps/provisioning/ansible/centos-network-setup/templates/ifcfg-interface b/snaps/provisioning/ansible/centos-network-setup/templates/ifcfg-interface
new file mode 100644
index 0000000..47aa3fa
--- /dev/null
+++ b/snaps/provisioning/ansible/centos-network-setup/templates/ifcfg-interface
@@ -0,0 +1,14 @@
+DEVICE={{ nic_name }}
+NAME={{ nic_name }}
+IPADDR={{ nic_ip }}
+
+DEFROUTE=no
+NETMASK=255.255.255.0
+NM_CONTROLLED=no
+IPV6INIT=yes
+IPV6_AUTOCONF=yes
+IPV6_DEFROUTE=yes
+IPV6_PEERDNS=yes
+IPV6_PEERROUTES=yes
+IPV6_FAILURE_FATAL=no
+ONBOOT=yes
\ No newline at end of file
diff --git a/snaps/provisioning/ansible/ubuntu-network-setup/playbooks/configure_host.yml b/snaps/provisioning/ansible/ubuntu-network-setup/playbooks/configure_host.yml
new file mode 100644
index 0000000..5d43f96
--- /dev/null
+++ b/snaps/provisioning/ansible/ubuntu-network-setup/playbooks/configure_host.yml
@@ -0,0 +1,26 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Configure NIC
+  hosts: all
+  become: yes
+  become_method: sudo
+  become_user: root
+
+  tasks:
+   - name: Setup /etc/network/interfaces.d/{{nic_name}}.cfg file
+     action: template owner=root group=root mode=644 src=../templates/ethN.cfg dest=/etc/network/interfaces.d/{{nic_name}}.cfg
+   - name : Restart Network
+     command: service networking restart
\ No newline at end of file
diff --git a/snaps/provisioning/ansible/ubuntu-network-setup/templates/ethN.cfg b/snaps/provisioning/ansible/ubuntu-network-setup/templates/ethN.cfg
new file mode 100644
index 0000000..3fa7708
--- /dev/null
+++ b/snaps/provisioning/ansible/ubuntu-network-setup/templates/ethN.cfg
@@ -0,0 +1,2 @@
+auto {{ nic_name }}
+iface {{ nic_name }} inet dhcp
diff --git a/snaps/provisioning/ansible_utils.py b/snaps/provisioning/ansible_utils.py
new file mode 100644
index 0000000..36f1efc
--- /dev/null
+++ b/snaps/provisioning/ansible_utils.py
@@ -0,0 +1,114 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from collections import namedtuple
+
+import os
+import paramiko
+
+from ansible.parsing.dataloader import DataLoader
+from ansible.vars import VariableManager
+from ansible.inventory import Inventory
+from ansible.executor.playbook_executor import PlaybookExecutor
+
+__author__ = 'spisarski'
+
+logger = logging.getLogger('ansible_utils')
+
+
+def apply_playbook(playbook_path, hosts_inv, host_user, ssh_priv_key_file_path, variables=None, proxy_setting=None):
+    """
+    Executes an Ansible playbook to the given host
+    :param playbook_path: the (relative) path to the Ansible playbook
+    :param hosts_inv: a list of hostnames/ip addresses to which to apply the Ansible playbook
+    :param host_user: A user for the host instances (must be a password-less sudo user if playbook has "sudo: yes"
+    :param ssh_priv_key_file_path: the file location of the ssh key
+    :param variables: a dictionary containing any substitution variables needed by the Jinga 2 templates
+    :param proxy_setting: instance of os_credentials.ProxySettings class
+    :return: the results
+    """
+    if not os.path.isfile(playbook_path):
+        raise Exception('Requested playbook not found - ' + playbook_path)
+    if not os.path.isfile(ssh_priv_key_file_path):
+        raise Exception('Requested private SSH key not found - ' + ssh_priv_key_file_path)
+
+    import ansible.constants
+    ansible.constants.HOST_KEY_CHECKING = False
+
+    variable_manager = VariableManager()
+    if variables:
+        variable_manager.extra_vars = variables
+
+    loader = DataLoader()
+    inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=hosts_inv)
+    variable_manager.set_inventory(inventory)
+
+    ssh_extra_args = None
+    if proxy_setting and proxy_setting.ssh_proxy_cmd:
+        ssh_extra_args = '-o ProxyCommand=\'' + proxy_setting.ssh_proxy_cmd + '\''
+
+    options = namedtuple('Options', ['listtags', 'listtasks', 'listhosts', 'syntax', 'connection', 'module_path',
+                                     'forks', 'remote_user', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
+                                     'become', 'become_method', 'become_user', 'verbosity', 'check'])
+
+    ansible_opts = options(listtags=False, listtasks=False, listhosts=False, syntax=False, connection='ssh',
+                           module_path=None, forks=100, remote_user=host_user, private_key_file=ssh_priv_key_file_path,
+                           ssh_common_args=None, ssh_extra_args=ssh_extra_args, become=None, become_method=None,
+                           become_user=None, verbosity=11111, check=False)
+
+    logger.debug('Setting up Ansible Playbook Executor for playbook - ' + playbook_path)
+    executor = PlaybookExecutor(
+        playbooks=[playbook_path],
+        inventory=inventory,
+        variable_manager=variable_manager,
+        loader=loader,
+        options=ansible_opts,
+        passwords=None)
+
+    logger.debug('Executing Ansible Playbook - ' + playbook_path)
+    retval = executor.run()
+
+    if retval != 0:
+        logger.error('Playbook application failed [' + playbook_path + '] with return value of - ' + str(retval))
+        raise Exception('Playbook not applied - ' + playbook_path)
+
+    return retval
+
+
+def ssh_client(ip, user, private_key_filepath, proxy_settings=None):
+    """
+    Retrieves and attemts an SSH connection
+    :param ip: the IP of the host to connect
+    :param user: the user with which to connect
+    :param private_key_filepath: the path to the private key file
+    :param proxy_settings: instance of os_credentials.ProxySettings class (optional)
+    :return: the SSH client if can connect else false
+    """
+    logger.debug('Retrieving SSH client')
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy())
+
+    try:
+        proxy_cmd = None
+        if proxy_settings and proxy_settings.ssh_proxy_cmd:
+            proxy_cmd_str = str(proxy_settings.ssh_proxy_cmd.replace('%h', ip))
+            proxy_cmd_str = proxy_cmd_str.replace("%p", '22')
+            proxy_cmd = paramiko.ProxyCommand(proxy_cmd_str)
+
+        ssh.connect(ip, username=user, key_filename=private_key_filepath, sock=proxy_cmd)
+        return ssh
+    except Exception as e:
+        logger.warn('Unable to connect via SSH with message - ' + e.message)
diff --git a/snaps/provisioning/tests/__init__.py b/snaps/provisioning/tests/__init__.py
new file mode 100644
index 0000000..e3e876e
--- /dev/null
+++ b/snaps/provisioning/tests/__init__.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+__author__ = 'spisarski'
diff --git a/snaps/provisioning/tests/ansible_utils_tests.py b/snaps/provisioning/tests/ansible_utils_tests.py
new file mode 100644
index 0000000..dc108e0
--- /dev/null
+++ b/snaps/provisioning/tests/ansible_utils_tests.py
@@ -0,0 +1,217 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import uuid
+
+from snaps.openstack import create_instance
+from snaps.openstack import create_keypairs
+from snaps.openstack import create_network
+from snaps.openstack import create_router
+from snaps.openstack import create_image
+from snaps.openstack import create_flavor
+from scp import SCPClient
+
+from snaps.provisioning import ansible_utils
+from snaps.openstack.tests import openstack_tests
+from snaps.openstack.tests.os_source_file_test import OSIntegrationTestCase
+
+VM_BOOT_TIMEOUT = 600
+
+ip_1 = '10.0.1.100'
+ip_2 = '10.0.1.200'
+
+
+class AnsibleProvisioningTests(OSIntegrationTestCase):
+    """
+    Test for the CreateInstance class with two NIC/Ports, eth0 with floating IP and eth1 w/o
+    """
+
+    def setUp(self):
+        """
+        Instantiates the CreateImage object that is responsible for downloading and creating an OS image file
+        within OpenStack
+        """
+        super(self.__class__, self).__start__()
+
+        guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
+        self.keypair_priv_filepath = 'tmp/' + guid
+        self.keypair_pub_filepath = self.keypair_priv_filepath + '.pub'
+        self.keypair_name = guid + '-kp'
+        self.vm_inst_name = guid + '-inst'
+        self.test_file_local_path = 'tmp/' + guid + '-hello.txt'
+        self.port_1_name = guid + '-port-1'
+        self.port_2_name = guid + '-port-2'
+        self.floating_ip_name = guid + 'fip1'
+
+        # Setup members to cleanup just in case they don't get created
+        self.inst_creator = None
+        self.keypair_creator = None
+        self.flavor_creator = None
+        self.router_creator = None
+        self.network_creator = None
+        self.image_creator = None
+
+        try:
+            # Create Image
+            os_image_settings = openstack_tests.ubuntu_url_image(name=guid + '-' + '-image')
+            self.image_creator = create_image.OpenStackImage(self.os_creds, os_image_settings)
+            self.image_creator.create()
+
+            # First network is public
+            self.pub_net_config = openstack_tests.get_pub_net_config(
+                net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet',
+                router_name=guid + '-pub-router', external_net=self.ext_net_name)
+
+            self.network_creator = create_network.OpenStackNetwork(self.os_creds, self.pub_net_config.network_settings)
+            self.network_creator.create()
+
+            # Create routers
+            self.router_creator = create_router.OpenStackRouter(self.os_creds, self.pub_net_config.router_settings)
+            self.router_creator.create()
+
+            # Create Flavor
+            self.flavor_creator = create_flavor.OpenStackFlavor(
+                self.admin_os_creds,
+                create_flavor.FlavorSettings(name=guid + '-flavor-name', ram=2048, disk=10, vcpus=2))
+            self.flavor_creator.create()
+
+            # Create Key/Pair
+            self.keypair_creator = create_keypairs.OpenStackKeypair(
+                self.os_creds, create_keypairs.KeypairSettings(
+                    name=self.keypair_name, public_filepath=self.keypair_pub_filepath,
+                    private_filepath=self.keypair_priv_filepath))
+            self.keypair_creator.create()
+
+            # Create instance
+            ports_settings = list()
+            ports_settings.append(
+                create_network.PortSettings(name=self.port_1_name,
+                                            network_name=self.pub_net_config.network_settings.name))
+
+            instance_settings = create_instance.VmInstanceSettings(
+                name=self.vm_inst_name, flavor=self.flavor_creator.flavor_settings.name, port_settings=ports_settings,
+                floating_ip_settings=[create_instance.FloatingIpSettings(
+                    name=self.floating_ip_name, port_name=self.port_1_name,
+                    router_name=self.pub_net_config.router_settings.name)])
+
+            self.inst_creator = create_instance.OpenStackVmInstance(
+                self.os_creds, instance_settings, self.image_creator.image_settings,
+                keypair_settings=self.keypair_creator.keypair_settings)
+        except Exception as e:
+            self.tearDown()
+            raise Exception(e.message)
+
+    def tearDown(self):
+        """
+        Cleans the created objects
+        """
+        if self.inst_creator:
+            self.inst_creator.clean()
+
+        if self.keypair_creator:
+            self.keypair_creator.clean()
+
+        if self.flavor_creator:
+            self.flavor_creator.clean()
+
+        if os.path.isfile(self.keypair_pub_filepath):
+            os.remove(self.keypair_pub_filepath)
+
+        if os.path.isfile(self.keypair_priv_filepath):
+            os.remove(self.keypair_priv_filepath)
+
+        if self.router_creator:
+            self.router_creator.clean()
+
+        if self.network_creator:
+            self.network_creator.clean()
+
+        if self.image_creator:
+            self.image_creator.clean()
+
+        if os.path.isfile(self.test_file_local_path):
+            os.remove(self.test_file_local_path)
+
+        super(self.__class__, self).__clean__()
+
+    def test_apply_simple_playbook(self):
+        """
+        Tests application of an Ansible playbook that simply copies over a file:
+        1. Have a ~/.ansible.cfg (or alternate means) to set host_key_checking = False
+        2. Set the following environment variable in your executing shell: ANSIBLE_HOST_KEY_CHECKING=False
+        Should this not be performed, the creation of the host ssh key will cause your ansible calls to fail.
+        """
+        self.inst_creator.create(block=True)
+
+        # Block until VM's ssh port has been opened
+        self.assertTrue(self.inst_creator.vm_ssh_active(block=True))
+
+        ssh_client = self.inst_creator.ssh_client()
+        self.assertIsNotNone(ssh_client)
+        out = ssh_client.exec_command('pwd')[1].channel.in_buffer.read(1024)
+        self.assertIsNotNone(out)
+        self.assertGreater(len(out), 1)
+
+        # Need to use the first floating IP as subsequent ones are currently broken with Apex CO
+        ip = self.inst_creator.get_floating_ip().ip
+        user = self.inst_creator.get_image_user()
+        priv_key = self.inst_creator.keypair_settings.private_filepath
+
+        retval = ansible_utils.apply_playbook('provisioning/tests/playbooks/simple_playbook.yml', [ip], user, priv_key,
+                                              proxy_setting=self.os_creds.proxy_settings)
+        self.assertEquals(0, retval)
+
+        ssh = ansible_utils.ssh_client(ip, user, priv_key, self.os_creds.proxy_settings)
+        self.assertIsNotNone(ssh)
+        scp = SCPClient(ssh.get_transport())
+        scp.get('~/hello.txt', self.test_file_local_path)
+
+        self.assertTrue(os.path.isfile(self.test_file_local_path))
+
+        with open(self.test_file_local_path) as f:
+            file_contents = f.readline()
+            self.assertEquals('Hello World!', file_contents)
+
+    def test_apply_template_playbook(self):
+        """
+        Tests application of an Ansible playbook that applies a template to a file:
+        1. Have a ~/.ansible.cfg (or alternate means) to set host_key_checking = False
+        2. Set the following environment variable in your executing shell: ANSIBLE_HOST_KEY_CHECKING=False
+        Should this not be performed, the creation of the host ssh key will cause your ansible calls to fail.
+        """
+        self.inst_creator.create(block=True)
+
+        # Block until VM's ssh port has been opened
+        self.assertTrue(self.inst_creator.vm_ssh_active(block=True))
+
+        # Need to use the first floating IP as subsequent ones are currently broken with Apex CO
+        ip = self.inst_creator.get_floating_ip().ip
+        user = self.inst_creator.get_image_user()
+        priv_key = self.inst_creator.keypair_settings.private_filepath
+
+        ansible_utils.apply_playbook('provisioning/tests/playbooks/template_playbook.yml', [ip], user, priv_key,
+                                     variables={'name': 'Foo'}, proxy_setting=self.os_creds.proxy_settings)
+
+        ssh = ansible_utils.ssh_client(ip, user, priv_key, self.os_creds.proxy_settings)
+        self.assertIsNotNone(ssh)
+        scp = SCPClient(ssh.get_transport())
+        scp.get('/tmp/hello.txt', self.test_file_local_path)
+
+        self.assertTrue(os.path.isfile(self.test_file_local_path))
+
+        with open(self.test_file_local_path) as f:
+            file_contents = f.readline()
+            self.assertEquals('Hello Foo!', file_contents)
diff --git a/snaps/provisioning/tests/playbooks/simple_playbook.yml b/snaps/provisioning/tests/playbooks/simple_playbook.yml
new file mode 100644
index 0000000..7af169c
--- /dev/null
+++ b/snaps/provisioning/tests/playbooks/simple_playbook.yml
@@ -0,0 +1,21 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- hosts: all
+
+  tasks:
+   - name: Transfer the test file
+     copy: src=../scripts/hello.txt dest=~/hello.txt mode=0777
+
diff --git a/snaps/provisioning/tests/playbooks/template_playbook.yml b/snaps/provisioning/tests/playbooks/template_playbook.yml
new file mode 100644
index 0000000..34d4e95
--- /dev/null
+++ b/snaps/provisioning/tests/playbooks/template_playbook.yml
@@ -0,0 +1,23 @@
+# Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- hosts: all
+  become: yes
+  become_method: sudo
+  become_user: root
+
+  tasks:
+   - name: Apply template and copy file
+     action: template owner=root group=root mode=777 src=../scripts/template.txt dest=/tmp/hello.txt
diff --git a/snaps/provisioning/tests/scripts/hello.txt b/snaps/provisioning/tests/scripts/hello.txt
new file mode 100644
index 0000000..c57eff5
--- /dev/null
+++ b/snaps/provisioning/tests/scripts/hello.txt
@@ -0,0 +1 @@
+Hello World!
\ No newline at end of file
diff --git a/snaps/provisioning/tests/scripts/template.txt b/snaps/provisioning/tests/scripts/template.txt
new file mode 100644
index 0000000..c7a43bc
--- /dev/null
+++ b/snaps/provisioning/tests/scripts/template.txt
@@ -0,0 +1 @@
+Hello {{ name }}!
\ No newline at end of file