Adds ability to deploy from upstream openstack

To deploy with upstream openstack branch, use new deploy setting 'os_version'.
A default scenario file for nosdn with pike has been included in this patch.
If 'os_version' is a version other than the default version for this OPNFV
release, then upstream is used.

In order to use upstream with the current OS version use '--upstream'
argument to the deploy command, to force an upstream deployment.

Also include '-e upstream-environment.yaml' to use default upstream
deployment settings.

Supports nosdn and odl-nofeature deployments.

Change-Id: Ic07e308827b449637b4e86cdd086434e4de2fb69
Signed-off-by: Tim Rozet <trozet@redhat.com>

6 files changed, 287 insertions, 2 deletions

diff --git a/apex/tests/test_apex_common_builder.py b/apex/tests/test_apex_common_builder.py
new file mode 100644
index 00000000..d042d2ba
--- /dev/null
+++ b/apex/tests/test_apex_common_builder.py
@@ -0,0 +1,86 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (Red Hat)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import unittest
+
+from apex.builders import common_builder as c_builder
+from apex.common import constants as con
+from mock import patch
+from mock import mock_open
+from mock import MagicMock
+
+
+class TestCommonBuilder(unittest.TestCase):
+    @classmethod
+    def setup_class(cls):
+        """This method is run once for each class before any tests are run"""
+
+    @classmethod
+    def teardown_class(cls):
+        """This method is run once for each class _after_ all tests are run"""
+
+    def setup(self):
+        """This method is run once before _each_ test method is executed"""
+
+    def teardown(self):
+        """This method is run once after _each_ test method is executed"""
+
+    def test_project_to_path(self):
+        project = 'openstack/tripleo-heat-templates'
+        path = '/usr/share/openstack-tripleo-heat-templates'
+        self.assertEquals(c_builder.project_to_path(project), path)
+        project = 'openstack/puppet-tripleo'
+        path = '/etc/puppet/modules/tripleo'
+        self.assertEquals(c_builder.project_to_path(project), path)
+        project = 'openstack/nova'
+        path = '/usr/lib/python2.7/site-packages/nova'
+        self.assertEquals(c_builder.project_to_path(project), path)
+
+    @patch('builtins.open', mock_open())
+    @patch('apex.build_utils.get_patch')
+    @patch('apex.virtual.utils.virt_customize')
+    def test_add_upstream_patches(self, mock_customize, mock_get_patch):
+        mock_get_patch.return_value = None
+        change_id = 'I301370fbf47a71291614dd60e4c64adc7b5ebb42'
+        patches = [{
+            'change-id': change_id,
+            'project': 'openstack/tripleo-heat-templates'
+        }]
+        c_builder.add_upstream_patches(patches, 'dummy.qcow2', '/dummytmp/')
+        assert mock_customize.not_called
+        project_path = '/usr/share/openstack-tripleo-heat-templates'
+        patch_file = "{}.patch".format(change_id)
+        patch_file_path = "/dummytmp/{}".format(patch_file)
+        test_virt_ops = [
+            {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
+                                             project_path)},
+            {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
+                project_path, patch_file)}]
+        mock_get_patch.return_value = 'some random diff'
+        c_builder.add_upstream_patches(patches, 'dummy.qcow2', '/dummytmp/')
+        mock_customize.assert_called_once_with(test_virt_ops, 'dummy.qcow2')
+
+    @patch('builtins.open', mock_open())
+    @patch('apex.virtual.utils.virt_customize')
+    def test_add_repo(self, mock_customize):
+        c_builder.add_repo('fake/url', 'dummyrepo', 'dummy.qcow2',
+                           '/dummytmp/')
+        repo_file_path = '/dummytmp/dummyrepo.repo'
+        test_virt_ops = [
+            {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}
+        ]
+        mock_customize.assert_called_once_with(test_virt_ops, 'dummy.qcow2')
+
+    @patch('builtins.open', mock_open())
+    @patch('git.Repo.clone_from')
+    def test_create_git_archive(self, mock_git):
+        mock_git.return_value = MagicMock()
+        self.assertEqual(c_builder.create_git_archive('fake/url', 'dummyrepo',
+                                                      '/dummytmp/'),
+                         '/dummytmp/dummyrepo.tar')
diff --git a/apex/tests/test_apex_common_utils.py b/apex/tests/test_apex_common_utils.py
index aee39a75..6f2a9476 100644
--- a/apex/tests/test_apex_common_utils.py
+++ b/apex/tests/test_apex_common_utils.py
@@ -9,6 +9,8 @@
 
 import ipaddress
 import os
+import shutil
+import urllib.error
 
 from apex.common import utils
 from apex.settings.network_settings import NetworkSettings
@@ -66,3 +68,35 @@ class TestCommonUtils:
         playbook = 'apex/tests/playbooks/test_failed_playbook.yaml'
         assert_raises(Exception, utils.run_ansible, None,
                       os.path.join(playbook), dry_run=True)
+
+    def test_fetch_upstream_and_unpack(self):
+        url = 'https://github.com/opnfv/apex/blob/master/'
+        utils.fetch_upstream_and_unpack('/tmp/fetch_test',
+                                        url, ['INFO'])
+        assert os.path.isfile('/tmp/fetch_test/INFO')
+        shutil.rmtree('/tmp/fetch_test')
+
+    def test_fetch_upstream_previous_file(self):
+        test_file = 'overcloud-full.tar.md5'
+        url = 'https://images.rdoproject.org/master/delorean/' \
+              'current-tripleo/stable/'
+        os.makedirs('/tmp/fetch_test', exist_ok=True)
+        open("/tmp/fetch_test/{}".format(test_file), 'w').close()
+        utils.fetch_upstream_and_unpack('/tmp/fetch_test',
+                                        url, [test_file])
+        assert os.path.isfile("/tmp/fetch_test/{}".format(test_file))
+        shutil.rmtree('/tmp/fetch_test')
+
+    def test_fetch_upstream_invalid_url(self):
+        url = 'http://notavalidsite.com/'
+        assert_raises(urllib.error.URLError,
+                      utils.fetch_upstream_and_unpack, '/tmp/fetch_test',
+                      url, ['INFO'])
+        shutil.rmtree('/tmp/fetch_test')
+
+    def test_fetch_upstream_and_unpack_tarball(self):
+        url = 'http://artifacts.opnfv.org/apex/tests/'
+        utils.fetch_upstream_and_unpack('/tmp/fetch_test',
+                                        url, ['dummy_test.tar'])
+        assert os.path.isfile('/tmp/fetch_test/test.txt')
+        shutil.rmtree('/tmp/fetch_test')
diff --git a/apex/tests/test_apex_deploy.py b/apex/tests/test_apex_deploy.py
index 4a0c983d..0a9b6c12 100644
--- a/apex/tests/test_apex_deploy.py
+++ b/apex/tests/test_apex_deploy.py
@@ -16,6 +16,7 @@ from mock import MagicMock
 from mock import mock_open
 
 from apex.common.exceptions import ApexDeployException
+from apex.common.constants import DEFAULT_OS_VERSION
 from apex.deploy import deploy_quickstart
 from apex.deploy import validate_cross_settings
 from apex.deploy import build_vms
@@ -140,11 +141,13 @@ class TestDeploy(unittest.TestCase):
                                            'dataplane': 'ovs',
                                            'sfc': False,
                                            'vpn': False,
-                                           'yardstick': 'test'}}
+                                           'yardstick': 'test',
+                                           'os_version': DEFAULT_OS_VERSION}}
         args = mock_parser.return_value.parse_args.return_value
         args.virtual = False
         args.quickstart = False
         args.debug = False
+        args.upstream = False
         net_sets = mock_net_sets.return_value
         net_sets.enabled_network_list = ['external']
         net_sets.__getitem__.side_effect = net_sets_dict.__getitem__
@@ -210,7 +213,8 @@ class TestDeploy(unittest.TestCase):
                                            'dataplane': 'ovs',
                                            'sfc': False,
                                            'vpn': False,
-                                           'yardstick': 'test'}}
+                                           'yardstick': 'test',
+                                           'os_version': DEFAULT_OS_VERSION}}
         args = mock_parser.return_value.parse_args.return_value
         args.virtual = True
         args.quickstart = False
@@ -220,6 +224,7 @@ class TestDeploy(unittest.TestCase):
         args.virt_compute_nodes = 1
         args.virt_compute_ram = None
         args.virt_default_ram = 12
+        args.upstream = False
         net_sets = mock_net_sets.return_value
         net_sets.enabled_network_list = ['admin']
         deploy_sets = mock_deploy_sets.return_value
diff --git a/apex/tests/test_apex_network_data.py b/apex/tests/test_apex_network_data.py
new file mode 100644
index 00000000..9197e1a8
--- /dev/null
+++ b/apex/tests/test_apex_network_data.py
@@ -0,0 +1,76 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (Red Hat)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+
+from nose.tools import (
+    assert_equal,
+    assert_is_instance,
+    assert_raises
+)
+
+from apex.common.constants import (
+    EXTERNAL_NETWORK,
+    STORAGE_NETWORK,
+    ADMIN_NETWORK,
+)
+from apex import NetworkSettings
+from apex.network import network_data
+from apex.settings.network_settings import NetworkSettingsException
+from apex.tests.constants import TEST_CONFIG_DIR
+
+files_dir = os.path.join(TEST_CONFIG_DIR, 'network')
+
+REQUIRED_KEYS = [
+    'name',
+    'vip',
+    'name_lower',
+    'enabled'
+]
+
+
+class TestNetworkData:
+    @classmethod
+    def setup_class(cls):
+        """This method is run once for each class before any tests are run"""
+
+    @classmethod
+    def teardown_class(cls):
+        """This method is run once for each class _after_ all tests are run"""
+
+    def setup(self):
+        """This method is run once before _each_ test method is executed"""
+
+    def teardown(self):
+        """This method is run once after _each_ test method is executed"""
+
+    def test_create_network_data(self):
+        ns = NetworkSettings(os.path.join(files_dir, 'network_settings.yaml'))
+        output = network_data.create_network_data(ns)
+        assert_is_instance(output, list)
+        # TODO(trozet) change this back to 4 after OOO bug is fixed
+        assert len(output) is 5
+        for net in output:
+            assert_is_instance(net, dict)
+            for key in REQUIRED_KEYS:
+                assert key in net
+                if key == 'vip' or key == 'enabled':
+                    assert_is_instance(net[key], bool)
+                else:
+                    assert net[key] is not None
+
+    def test_negative_create_network_data(self):
+        assert_raises(network_data.NetworkDataException,
+                      network_data.create_network_data, 'blah')
+
+    def test_create_network_data_with_write(self):
+        ns = NetworkSettings(os.path.join(files_dir, 'network_settings.yaml'))
+        network_data.create_network_data(ns, '/tmp/blah_network_data.yaml')
+        assert os.path.isfile('/tmp/blah_network_data.yaml')
+        os.remove('/tmp/blah_network_data.yaml')
diff --git a/apex/tests/test_apex_overcloud_builder.py b/apex/tests/test_apex_overcloud_builder.py
new file mode 100644
index 00000000..e9a6e6cf
--- /dev/null
+++ b/apex/tests/test_apex_overcloud_builder.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (Red Hat)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import unittest
+
+from apex.builders import overcloud_builder as oc_builder
+from apex.common import constants as con
+from mock import patch
+
+
+class TestOvercloudBuilder(unittest.TestCase):
+    @classmethod
+    def setup_class(cls):
+        """This method is run once for each class before any tests are run"""
+
+    @classmethod
+    def teardown_class(cls):
+        """This method is run once for each class _after_ all tests are run"""
+
+    def setup(self):
+        """This method is run once before _each_ test method is executed"""
+
+    def teardown(self):
+        """This method is run once after _each_ test method is executed"""
+
+    @patch('apex.builders.common_builder.create_git_archive')
+    @patch('apex.builders.common_builder.add_repo')
+    @patch('apex.virtual.utils.virt_customize')
+    def test_inject_opendaylight(self, mock_customize, mock_add_repo,
+                                 mock_git_archive):
+        mock_git_archive.return_value = '/dummytmp/puppet-opendaylight.tar'
+        archive = '/dummytmp/puppet-opendaylight.tar'
+        test_virt_ops = [
+            {con.VIRT_INSTALL: 'opendaylight'},
+            {con.VIRT_UPLOAD: "{}:/etc/puppet/modules/".format(archive)},
+            {con.VIRT_RUN_CMD: 'rm -rf /etc/puppet/modules/opendaylight'},
+            {con.VIRT_RUN_CMD: "cd /etc/puppet/modules/ && tar xvf "
+                               "puppet-opendaylight.tar"}
+        ]
+        oc_builder.inject_opendaylight(con.DEFAULT_ODL_VERSION, 'dummy.qcow2',
+                                       '/dummytmp/')
+        assert mock_git_archive.called
+        assert mock_add_repo.called
+        mock_customize.assert_called_once_with(test_virt_ops, 'dummy.qcow2')
diff --git a/apex/tests/test_apex_undercloud_builder.py b/apex/tests/test_apex_undercloud_builder.py
new file mode 100644
index 00000000..c7498880
--- /dev/null
+++ b/apex/tests/test_apex_undercloud_builder.py
@@ -0,0 +1,34 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (Red Hat)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import unittest
+
+from apex.builders import undercloud_builder as uc_builder
+from mock import patch
+
+
+class TestUndercloudBuilder(unittest.TestCase):
+    @classmethod
+    def setup_class(cls):
+        """This method is run once for each class before any tests are run"""
+
+    @classmethod
+    def teardown_class(cls):
+        """This method is run once for each class _after_ all tests are run"""
+
+    def setup(self):
+        """This method is run once before _each_ test method is executed"""
+
+    def teardown(self):
+        """This method is run once after _each_ test method is executed"""
+
+    @patch('apex.virtual.utils.virt_customize')
+    def test_add_upstream_pkgs(self, mock_customize):
+        uc_builder.add_upstream_packages('dummy.qcow2')
+        assert mock_customize.called