Allow common patches file

This patch adds allowing for common patches that should be applied to
every scenario to be included. It by default pulls in a file in the
deploy directory 'common-patches.yaml', but can optionally be
overridden.

This patch also includes a patch upstream to fix OSCLI not working
anymore due to breakage with the Cinder version in the overcloudrc.

Change-Id: I97b9efb937deff07e085b9ef75b9799fb65bfc57
Signed-off-by: Tim Rozet <trozet@redhat.com>

15 files changed, 174 insertions, 20 deletions

diff --git a/apex/common/utils.py b/apex/common/utils.py
index 013c7ac8..464aaf28 100644
--- a/apex/common/utils.py
+++ b/apex/common/utils.py
@@ -272,3 +272,12 @@ def edit_tht_env(env_file, section, settings):
     with open(env_file, 'w') as fh:
         yaml.safe_dump(data, fh, default_flow_style=False)
     logging.debug("Data written to env file {}:\n{}".format(env_file, data))
+
+
+def unique(tmp_list):
+    assert isinstance(tmp_list, list)
+    uniq_list = []
+    for x in tmp_list:
+        if x not in uniq_list:
+            uniq_list.append(x)
+    return uniq_list
diff --git a/apex/deploy.py b/apex/deploy.py
index ca4101b4..8065d5ca 100644
--- a/apex/deploy.py
+++ b/apex/deploy.py
@@ -34,6 +34,7 @@ from apex.common import utils
 from apex.common import constants
 from apex.common import parsers
 from apex.common.exceptions import ApexDeployException
+from apex.deployment.tripleo import ApexDeployment
 from apex.network import jumphost
 from apex.network import network_data
 from apex.undercloud import undercloud as uc_lib
@@ -188,6 +189,11 @@ def create_deploy_parser():
                                default=False,
                                help='Ignore fetching latest upstream and '
                                     'use what is in cache')
+    deploy_parser.add_argument('-p', '--patches',
+                               default='/etc/opnfv-apex/common-patches.yaml',
+                               dest='patches_file',
+                               help='File to include for common patches '
+                                    'which apply to all deployment scenarios')
     return deploy_parser
 
 
@@ -308,6 +314,8 @@ def main():
         deploy_quickstart(args, deploy_settings_file, network_settings_file,
                           args.inventory_file)
     else:
+        deployment = ApexDeployment(deploy_settings, args.patches_file,
+                                    args.deploy_settings_file)
         # TODO (trozet): add logic back from:
         # Iedb75994d35b5dc1dd5d5ce1a57277c8f3729dfd (FDIO DVR)
         ansible_args = {
@@ -373,7 +381,7 @@ def main():
         uc_builder.add_upstream_packages(uc_image)
         # add patches from upstream to undercloud and overcloud
         logging.info('Adding patches to undercloud')
-        patches = deploy_settings['global_params']['patches']
+        patches = deployment.determine_patches()
         c_builder.add_upstream_patches(patches['undercloud'], uc_image,
                                        APEX_TEMP_DIR, branch)
 
diff --git a/apex/deployment/__init__.py b/apex/deployment/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/apex/deployment/__init__.py
diff --git a/apex/deployment/tripleo.py b/apex/deployment/tripleo.py
new file mode 100644
index 00000000..0f85bbae
--- /dev/null
+++ b/apex/deployment/tripleo.py
@@ -0,0 +1,59 @@
+##############################################################################
+# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# TODO(trozet): this will serve as the deployment class as we migrate logic out
+# of deploy.py
+import logging
+import os
+import pprint
+
+from apex.common.exceptions import ApexDeployException
+from apex.common import utils
+
+
+class ApexDeployment:
+    def __init__(self, deploy_settings, patch_file, ds_file):
+        self.ds = deploy_settings
+        # TODO(trozet): remove ds_file from args and have this class inherit
+        # super deployment class init which does all the settings
+        self.ds_file = ds_file
+        self.ds_globals = self.ds['global_params']
+        self.p_file = patch_file
+
+    def determine_patches(self):
+        patches = self.ds_globals['patches']
+        if not os.path.isfile(self.p_file):
+            new_file = os.path.join(os.path.dirname(self.ds_file),
+                                    'common-patches.yaml')
+            if os.path.isfile(new_file):
+                logging.warning('Patch file {} not found, falling back to '
+                                '{}'.format(self.p_file, new_file))
+                self.p_file = new_file
+            else:
+                logging.error('Unable to find common patch file: '
+                              '{}'.format(self.p_file))
+                raise ApexDeployException(
+                    'Specified common patch file not found: {}'.format(
+                        self.p_file))
+        logging.info('Loading patches from common patch file {}'.format(
+            self.p_file))
+        common_patches = utils.parse_yaml(self.p_file)
+        logging.debug('Content from common patch file is: {}'.format(
+            pprint.pformat(common_patches)))
+        if 'patches' not in common_patches.keys():
+            logging.error('Error parsing common patches file, wrong format. '
+                          'Missing "patches" dictionary')
+            raise ApexDeployException('Invalid format of common patch file')
+        else:
+            common_patches = common_patches['patches']
+        for ptype in ('undercloud', 'overcloud'):
+            if ptype in common_patches:
+                patches[ptype] = utils.unique(patches[ptype] +
+                                              common_patches[ptype])
+        return patches
diff --git a/apex/tests/config/common-patches.yaml b/apex/tests/config/common-patches.yaml
new file mode 100644
index 00000000..35dbf449
--- /dev/null
+++ b/apex/tests/config/common-patches.yaml
@@ -0,0 +1,5 @@
+---
+patches:
+  undercloud:
+    - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
+      project: openstack/tripleo-common
diff --git a/apex/tests/config/dummy-deploy-settings.yaml b/apex/tests/config/dummy-deploy-settings.yaml
new file mode 100644
index 00000000..54890f38
--- /dev/null
+++ b/apex/tests/config/dummy-deploy-settings.yaml
@@ -0,0 +1,19 @@
+---
+global_params:
+  ha_enabled: false
+  patches:
+    undercloud:
+      - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
+        project: openstack/tripleo-common
+    overcloud:
+      - change-id: Ie988ba6a2d444a614e97c0edf5fce24b23970310
+        project: openstack/puppet-tripleo
+deploy_options:
+  containers: true
+  os_version: queens
+  sdn_controller: opendaylight
+  odl_version: oxygen
+  tacker: false
+  congress: false
+  sfc: false
+  vpn: false
diff --git a/apex/tests/test_apex_common_utils.py b/apex/tests/test_apex_common_utils.py
index b6aa4c7f..412d6f49 100644
--- a/apex/tests/test_apex_common_utils.py
+++ b/apex/tests/test_apex_common_utils.py
@@ -151,3 +151,7 @@ class TestCommonUtils:
         new_data = {'parameter_defaults': settings}
         mock_yaml_dump.assert_called_once_with(new_data, a_mock_open(),
                                                default_flow_style=False)
+
+    def test_unique(self):
+        dummy_list = [1, 2, 1, 3, 4, 5, 5]
+        assert_equal(utils.unique(dummy_list), [1, 2, 3, 4, 5])
diff --git a/apex/tests/test_apex_deploy.py b/apex/tests/test_apex_deploy.py
index 8e9756eb..5741818a 100644
--- a/apex/tests/test_apex_deploy.py
+++ b/apex/tests/test_apex_deploy.py
@@ -107,6 +107,7 @@ class TestDeploy(unittest.TestCase):
         args.virtual = True
         assert_raises(ApexDeployException, validate_deploy_args, args)
 
+    @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.network_data.create_network_data')
     @patch('apex.deploy.shutil')
@@ -134,7 +135,7 @@ class TestDeploy(unittest.TestCase):
                   mock_utils, mock_parsers, mock_oc_cfg,
                   mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
                   mock_oc_deploy, mock_shutil, mock_network_data,
-                  mock_uc_builder):
+                  mock_uc_builder, mock_deployment):
         net_sets_dict = {'networks': MagicMock(),
                          'dns_servers': 'test'}
         ds_opts_dict = {'global_params': MagicMock(),
@@ -182,6 +183,7 @@ class TestDeploy(unittest.TestCase):
         args.debug = True
         main()
 
+    @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.network_data.create_network_data')
     @patch('apex.deploy.shutil')
@@ -209,7 +211,7 @@ class TestDeploy(unittest.TestCase):
                        mock_utils, mock_parsers, mock_oc_cfg,
                        mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
                        mock_oc_deploy, mock_shutil, mock_network_data,
-                       mock_uc_builder):
+                       mock_uc_builder, mock_deployment):
         # didn't work yet line 412
         # net_sets_dict = {'networks': {'admin': {'cidr': MagicMock()}},
         #                 'dns_servers': 'test'}
@@ -245,6 +247,7 @@ class TestDeploy(unittest.TestCase):
         args.virt_default_ram = 10
         main()
 
+    @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.c_builder')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.oc_builder')
@@ -274,7 +277,7 @@ class TestDeploy(unittest.TestCase):
             mock_utils, mock_parsers, mock_oc_cfg, mock_virt_utils,
             mock_inv, mock_build_vms, mock_uc_lib, mock_oc_deploy,
             mock_shutil, mock_network_data, mock_oc_builder,
-            mock_uc_builder, mock_c_builder):
+            mock_uc_builder, mock_c_builder, mock_deployment):
 
         ds_opts_dict = {'global_params': MagicMock(),
                         'deploy_options': {'gluon': False,
@@ -310,6 +313,7 @@ class TestDeploy(unittest.TestCase):
         # TODO(trozet) add assertions here with arguments for functions in
         # deploy main
 
+    @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.network_data.create_network_data')
     @patch('apex.deploy.shutil')
@@ -338,7 +342,7 @@ class TestDeploy(unittest.TestCase):
                       mock_utils, mock_parsers, mock_oc_cfg,
                       mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
                       mock_oc_deploy, mock_git, mock_shutil,
-                      mock_network_data, mock_uc_builder):
+                      mock_network_data, mock_uc_builder, mock_deployment):
         net_sets_dict = {'networks': MagicMock(),
                          'dns_servers': 'test'}
         ds_opts_dict = {'global_params': MagicMock(),
diff --git a/apex/tests/test_apex_deployment_tripleo.py b/apex/tests/test_apex_deployment_tripleo.py
new file mode 100644
index 00000000..912fe104
--- /dev/null
+++ b/apex/tests/test_apex_deployment_tripleo.py
@@ -0,0 +1,49 @@
+##############################################################################
+# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) (Red Hat)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+import unittest
+
+from apex.deployment.tripleo import ApexDeployment
+from apex.settings.deploy_settings import DeploySettings
+from apex.tests.constants import TEST_DUMMY_CONFIG
+
+
+class TestApexDeployment(unittest.TestCase):
+    @classmethod
+    def setup_class(cls):
+        """This method is run once for each class before any tests are run"""
+
+    @classmethod
+    def teardown_class(cls):
+        """This method is run once for each class _after_ all tests are run"""
+
+    def setup(self):
+        """This method is run once before _each_ test method is executed"""
+
+    def teardown(self):
+        """This method is run once after _each_ test method is executed"""
+
+    def test_determine_patches(self):
+        self.maxDiff = None
+        ds_file = os.path.join(TEST_DUMMY_CONFIG, 'dummy-deploy-settings.yaml')
+        ds = DeploySettings(ds_file)
+        patches_file = os.path.join(TEST_DUMMY_CONFIG, 'common-patches.yaml')
+        d = ApexDeployment(deploy_settings=ds, patch_file=patches_file,
+                           ds_file=ds_file)
+        patches = d.determine_patches()
+        test_patches = {
+            'undercloud':
+                [{'change-id': 'I2e0a40d7902f592e4b7bd727f57048111e0bea36',
+                  'project': 'openstack/tripleo-common'}],
+            'overcloud':
+                [{'change-id': 'Ie988ba6a2d444a614e97c0edf5fce24b23970310',
+                  'project': 'openstack/puppet-tripleo'}]
+        }
+        self.assertDictEqual(patches, test_patches)
diff --git a/build/rpm_specs/opnfv-apex.spec b/build/rpm_specs/opnfv-apex.spec
index c344dfda..0f6f1f84 100644
--- a/build/rpm_specs/opnfv-apex.spec
+++ b/build/rpm_specs/opnfv-apex.spec
@@ -67,6 +67,7 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %attr(755,root,root) %{_bindir}/opnfv-pyutil
 %{_datadir}/opnfv-apex/
 %{_sysconfdir}/bash_completion.d/apex
+%{_sysconfdir}/opnfv-apex/common-patches.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-bar-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-bar-ha.yaml
@@ -125,7 +126,9 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %doc %{_docdir}/opnfv/inventory.yaml.example
 
 %changelog
-* Wed Jun 27 2018 Feng Pan <fpan@redhat.com> -7.0-4
+* Tue Aug 14 2018 Tim Rozet <trozet@redhat.com> - 7.0-5
+  Adds common patches file
+* Wed Jun 27 2018 Feng Pan <fpan@redhat.com> - 7.0-4
   Adds network_settings_tenant_vlan.yaml
 * Wed Jun 20 2018 Zenghui Shi <zshi@redhat.com> - 7.0-3
   Adds Kubernetes deployment scenario
diff --git a/config/deploy/common-patches.yaml b/config/deploy/common-patches.yaml
new file mode 100644
index 00000000..7339e79c
--- /dev/null
+++ b/config/deploy/common-patches.yaml
@@ -0,0 +1,8 @@
+---
+patches:
+  undercloud:
+    - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
+      project: openstack/tripleo-common
+    - change-id: Iaa2276aadae351fbc138de258c51d786f69e4395
+      project: openstack/tripleo-common
+      branch: master
diff --git a/config/deploy/os-nosdn-nofeature-ha.yaml b/config/deploy/os-nosdn-nofeature-ha.yaml
index d642533b..26d30e5e 100644
--- a/config/deploy/os-nosdn-nofeature-ha.yaml
+++ b/config/deploy/os-nosdn-nofeature-ha.yaml
@@ -1,10 +1,6 @@
 ---
 global_params:
   ha_enabled: true
-  patches:
-    undercloud:
-      - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
-        project: openstack/tripleo-common
 deploy_options:
   containers: true
   os_version: master
diff --git a/config/deploy/os-nosdn-nofeature-noha.yaml b/config/deploy/os-nosdn-nofeature-noha.yaml
index 41b9ec72..e7758113 100644
--- a/config/deploy/os-nosdn-nofeature-noha.yaml
+++ b/config/deploy/os-nosdn-nofeature-noha.yaml
@@ -1,10 +1,6 @@
 ---
 global_params:
   ha_enabled: false
-  patches:
-    undercloud:
-      - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
-        project: openstack/tripleo-common
 deploy_options:
   containers: true
   os_version: master
diff --git a/config/deploy/os-odl-nofeature-ha.yaml b/config/deploy/os-odl-nofeature-ha.yaml
index a46b4849..a1f0d81e 100644
--- a/config/deploy/os-odl-nofeature-ha.yaml
+++ b/config/deploy/os-odl-nofeature-ha.yaml
@@ -2,9 +2,6 @@
 global_params:
   ha_enabled: true
   patches:
-    undercloud:
-      - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
-        project: openstack/tripleo-common
     overcloud:
       - change-id: Ie988ba6a2d444a614e97c0edf5fce24b23970310
         project: openstack/puppet-tripleo
diff --git a/config/deploy/os-odl-nofeature-noha.yaml b/config/deploy/os-odl-nofeature-noha.yaml
index 609a03da..55f14426 100644
--- a/config/deploy/os-odl-nofeature-noha.yaml
+++ b/config/deploy/os-odl-nofeature-noha.yaml
@@ -2,9 +2,6 @@
 global_params:
   ha_enabled: false
   patches:
-    undercloud:
-      - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36
-        project: openstack/tripleo-common
     overcloud:
       - change-id: Ie988ba6a2d444a614e97c0edf5fce24b23970310
         project: openstack/puppet-tripleo