Merge "Fixes Docker image upload for master/rocky"

12 files changed, 153 insertions, 53 deletions

diff --git a/apex/builders/common_builder.py b/apex/builders/common_builder.py
index b8894ec1..e0e7303e 100644
--- a/apex/builders/common_builder.py
+++ b/apex/builders/common_builder.py
@@ -14,8 +14,11 @@ import git
 import json
 import logging
 import os
+import platform
+import pprint
 import re
 import urllib.parse
+import yaml
 
 import apex.builders.overcloud_builder as oc_builder
 from apex import build_utils
@@ -258,3 +261,39 @@ def create_git_archive(repo_url, repo_name, tmp_dir,
         repo.archive(fh, prefix=prefix)
     logging.debug("Wrote archive file: {}".format(archive_path))
     return archive_path
+
+
+def get_neutron_driver(ds_opts):
+    sdn = ds_opts.get('sdn_controller', None)
+    for controllers in 'opendaylight', 'ovn':
+        if sdn == controllers:
+            return sdn
+
+    if ds_opts.get('vpp', False):
+        return 'vpp'
+
+    return None
+
+
+def prepare_container_images(prep_file, branch='master', neutron_driver=None):
+    if not os.path.isfile(prep_file):
+        raise exc.ApexCommonBuilderException("Prep file does not exist: "
+                                             "{}".format(prep_file))
+    with open(prep_file) as fh:
+        data = yaml.safe_load(fh)
+    try:
+        p_set = data['parameter_defaults']['ContainerImagePrepare'][0]['set']
+        if neutron_driver:
+            p_set['neutron_driver'] = neutron_driver
+        p_set['namespace'] = "docker.io/tripleo{}".format(branch)
+        if platform.machine() == 'aarch64':
+            p_set['ceph_tag'] = 'master-fafda7d-luminous-centos-7-aarch64'
+
+    except KeyError:
+        logging.error("Invalid prep file format: {}".format(prep_file))
+        raise exc.ApexCommonBuilderException("Invalid format for prep file")
+
+    logging.debug("Writing new container prep file:\n{}".format(
+        pprint.pformat(data)))
+    with open(prep_file, 'w') as fh:
+        yaml.safe_dump(data, fh, default_flow_style=False)
diff --git a/apex/deploy.py b/apex/deploy.py
index dab6bd1e..f1ab260f 100644
--- a/apex/deploy.py
+++ b/apex/deploy.py
@@ -415,6 +415,15 @@ def main():
         for role in 'compute', 'controller':
             oc_cfg.create_nic_template(net_settings, deploy_settings, role,
                                        args.deploy_dir, APEX_TEMP_DIR)
+        # Prepare/Upload docker images
+        docker_env = 'containers-prepare-parameter.yaml'
+        shutil.copyfile(os.path.join(args.deploy_dir, docker_env),
+                        os.path.join(APEX_TEMP_DIR, docker_env))
+        c_builder.prepare_container_images(
+            os.path.join(APEX_TEMP_DIR, docker_env),
+            branch=branch,
+            neutron_driver=c_builder.get_neutron_driver(ds_opts)
+        )
         # Install Undercloud
         undercloud.configure(net_settings, deploy_settings,
                              os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
diff --git a/apex/overcloud/deploy.py b/apex/overcloud/deploy.py
index 4deeabff..c526a98e 100644
--- a/apex/overcloud/deploy.py
+++ b/apex/overcloud/deploy.py
@@ -194,8 +194,6 @@ def create_deploy_cmd(ds, ns, inv, tmp_dir,
         sdn_docker_files = get_docker_sdn_files(ds_opts)
         for sdn_docker_file in sdn_docker_files:
             deploy_options.append(sdn_docker_file)
-        if sdn_docker_files:
-            deploy_options.append('sdn-images.yaml')
     else:
         deploy_options += build_sdn_env_list(ds_opts, SDN_FILE_MAP)
 
diff --git a/apex/tests/test_apex_common_builder.py b/apex/tests/test_apex_common_builder.py
index 09bd2545..dede55a2 100644
--- a/apex/tests/test_apex_common_builder.py
+++ b/apex/tests/test_apex_common_builder.py
@@ -24,6 +24,8 @@ DOCKER_YAML = {
     }
 }
 
+a_mock_open = mock_open(read_data=None)
+
 
 class TestCommonBuilder(unittest.TestCase):
     @classmethod
@@ -249,3 +251,53 @@ class TestCommonBuilder(unittest.TestCase):
         self.assertRaises(exceptions.ApexCommonBuilderException,
                           c_builder.project_to_docker_image,
                           'nova')
+
+    def test_get_neutron_driver(self):
+        ds_opts = {'dataplane': 'fdio',
+                   'sdn_controller': 'opendaylight',
+                   'odl_version': 'master',
+                   'vpn': False,
+                   'sriov': False}
+        self.assertEquals(c_builder.get_neutron_driver(ds_opts),
+                          'opendaylight')
+        ds_opts['sdn_controller'] = None
+        ds_opts['vpp'] = True
+        self.assertEquals(c_builder.get_neutron_driver(ds_opts),
+                          'vpp')
+        ds_opts['sdn_controller'] = 'ovn'
+        self.assertEquals(c_builder.get_neutron_driver(ds_opts),
+                          'ovn')
+
+    @patch('apex.builders.common_builder.yaml')
+    @patch('apex.overcloud.deploy.os.path.isfile')
+    @patch('builtins.open', a_mock_open, create=True)
+    def test_prepare_container_images(self, mock_is_file, mock_yaml):
+        mock_yaml.safe_load.return_value = {
+            'parameter_defaults': {
+                'ContainerImagePrepare': [
+                    {'set':
+                        {'namespace': 'blah',
+                         'neutron_driver': 'null',
+                         }
+                     }
+                ]
+            }
+        }
+        expected_output = {
+            'parameter_defaults': {
+                'ContainerImagePrepare': [
+                    {'set':
+                        {'namespace': 'docker.io/tripleoqueens',
+                         'neutron_driver': 'opendaylight',
+                         }
+                     }
+                ]
+            }
+        }
+
+        c_builder.prepare_container_images('dummy.yaml', 'queens',
+                                           'opendaylight')
+        mock_yaml.safe_dump.assert_called_with(
+            expected_output,
+            a_mock_open.return_value,
+            default_flow_style=False)
diff --git a/apex/tests/test_apex_deploy.py b/apex/tests/test_apex_deploy.py
index be52c276..004c21c1 100644
--- a/apex/tests/test_apex_deploy.py
+++ b/apex/tests/test_apex_deploy.py
@@ -118,6 +118,7 @@ class TestDeploy(unittest.TestCase):
         args.snapshot = False
         assert_raises(ApexDeployException, validate_deploy_args, args)
 
+    @patch('apex.deploy.c_builder')
     @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.network_data.create_network_data')
@@ -146,7 +147,7 @@ class TestDeploy(unittest.TestCase):
                   mock_utils, mock_parsers, mock_oc_cfg,
                   mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
                   mock_oc_deploy, mock_shutil, mock_network_data,
-                  mock_uc_builder, mock_deployment):
+                  mock_uc_builder, mock_deployment, mock_c_builder):
         net_sets_dict = {'networks': MagicMock(),
                          'dns_servers': 'test'}
         ds_opts_dict = {'global_params': MagicMock(),
@@ -197,6 +198,7 @@ class TestDeploy(unittest.TestCase):
         main()
         mock_snap_deployment.assert_called()
 
+    @patch('apex.deploy.c_builder')
     @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.network_data.create_network_data')
@@ -225,7 +227,7 @@ class TestDeploy(unittest.TestCase):
                        mock_utils, mock_parsers, mock_oc_cfg,
                        mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
                        mock_oc_deploy, mock_shutil, mock_network_data,
-                       mock_uc_builder, mock_deployment):
+                       mock_uc_builder, mock_deployment, mock_c_builder):
         # didn't work yet line 412
         # net_sets_dict = {'networks': {'admin': {'cidr': MagicMock()}},
         #                 'dns_servers': 'test'}
@@ -329,6 +331,7 @@ class TestDeploy(unittest.TestCase):
         # TODO(trozet) add assertions here with arguments for functions in
         # deploy main
 
+    @patch('apex.deploy.c_builder')
     @patch('apex.deploy.ApexDeployment')
     @patch('apex.deploy.uc_builder')
     @patch('apex.deploy.network_data.create_network_data')
@@ -358,7 +361,8 @@ class TestDeploy(unittest.TestCase):
                       mock_utils, mock_parsers, mock_oc_cfg,
                       mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
                       mock_oc_deploy, mock_git, mock_shutil,
-                      mock_network_data, mock_uc_builder, mock_deployment):
+                      mock_network_data, mock_uc_builder, mock_deployment,
+                      mock_c_builder):
         net_sets_dict = {'networks': MagicMock(),
                          'dns_servers': 'test'}
         ds_opts_dict = {'global_params': MagicMock(),
diff --git a/apex/tests/test_apex_overcloud_deploy.py b/apex/tests/test_apex_overcloud_deploy.py
index a70057b9..402ecebf 100644
--- a/apex/tests/test_apex_overcloud_deploy.py
+++ b/apex/tests/test_apex_overcloud_deploy.py
@@ -156,7 +156,6 @@ class TestOvercloudDeploy(unittest.TestCase):
         assert_in('--control-scale 3', result_cmd)
         assert_in('--compute-scale 2', result_cmd)
         assert_in('docker-images.yaml', result_cmd)
-        assert_in('sdn-images.yaml', result_cmd)
         assert_in('/usr/share/openstack-tripleo-heat-templates/environments'
                   '/docker.yaml', result_cmd)
         assert_in('/usr/share/openstack-tripleo-heat-templates/environments/'
diff --git a/apex/undercloud/undercloud.py b/apex/undercloud/undercloud.py
index 1658801d..8b6b9d4c 100644
--- a/apex/undercloud/undercloud.py
+++ b/apex/undercloud/undercloud.py
@@ -235,7 +235,9 @@ class Undercloud:
             "local_ip {}/{}".format(str(ns_admin['installer_vm']['ip']),
                                     str(ns_admin['cidr']).split('/')[1]),
             "generate_service_certificate false",
-            "undercloud_ntp_servers {}".format(str(ns['ntp'][0]))
+            "undercloud_ntp_servers {}".format(str(ns['ntp'][0])),
+            "container_images_file "
+            "/home/stack/containers-prepare-parameter.yaml"
         ]
 
         config['undercloud_network_config'] = [
diff --git a/build/containers-prepare-parameter.yaml b/build/containers-prepare-parameter.yaml
new file mode 100644
index 00000000..d6cfd10a
--- /dev/null
+++ b/build/containers-prepare-parameter.yaml
@@ -0,0 +1,26 @@
+---
+parameter_defaults:
+  ContainerImagePrepare:
+    - push_destination: true
+      set:
+        ceph_image: daemon
+        ceph_namespace: docker.io/ceph
+        ceph_tag: v3.1.0-stable-3.1-luminous-centos-7-x86_64
+        name_prefix: centos-binary-
+        name_suffix: ''
+        namespace: docker.io/tripleomaster
+        neutron_driver: null
+        tag: current-tripleo-rdo
+      excludes:
+        - sensu
+        - manila
+        - octavia
+        - skydive
+        - drouter
+        - sahara
+        - rsys
+        - fluent
+        - designate
+        - barbican
+        - etcd
+        - ec2
diff --git a/config/deploy/common-patches.yaml b/config/deploy/common-patches.yaml
index c2172616..bac6812c 100644
--- a/config/deploy/common-patches.yaml
+++ b/config/deploy/common-patches.yaml
@@ -8,6 +8,8 @@ patches:
         project: openstack/tripleo-common
       - change-id: I8d1db69d520da069099f919f286e6a553dd645a5
         project: openstack/tripleo-heat-templates
+      - change-id: Ia51a825d11bd9b94d0110f13cdf2a6bbcedf6194
+        project: openstack/tripleo-common
     overcloud:
       - change-id: Ie988ba6a2d444a614e97c0edf5fce24b23970310
         project: openstack/puppet-tripleo
@@ -23,6 +25,8 @@ patches:
         branch: master
       - change-id: I8d1db69d520da069099f919f286e6a553dd645a5
         project: openstack/tripleo-heat-templates
+      - change-id: Ia51a825d11bd9b94d0110f13cdf2a6bbcedf6194
+        project: openstack/tripleo-common
     overcloud:
       - change-id: Ie988ba6a2d444a614e97c0edf5fce24b23970310
         project: openstack/puppet-tripleo
diff --git a/lib/ansible/playbooks/configure_undercloud.yml b/lib/ansible/playbooks/configure_undercloud.yml
index 5e11ea9f..45d18e46 100644
--- a/lib/ansible/playbooks/configure_undercloud.yml
+++ b/lib/ansible/playbooks/configure_undercloud.yml
@@ -27,6 +27,13 @@
       with_items:
         - controller
         - compute
+    - name: Copy container prep env file to undercloud
+      copy:
+        src: "{{ apex_temp_dir }}/containers-prepare-parameter.yaml"
+        dest: "/home/stack/containers-prepare-parameter.yaml"
+        owner: stack
+        group: stack
+        mode: 0644
     - lineinfile:
         path: /etc/sudoers
         regexp: 'Defaults\s*requiretty'
diff --git a/lib/ansible/playbooks/prepare_overcloud_containers.yml b/lib/ansible/playbooks/prepare_overcloud_containers.yml
index 54dbe098..e2a4e134 100644
--- a/lib/ansible/playbooks/prepare_overcloud_containers.yml
+++ b/lib/ansible/playbooks/prepare_overcloud_containers.yml
@@ -20,49 +20,9 @@
       when: patched_docker_services|length > 0
     - name: Prepare generic docker registry image file
       shell: >
-        {{ stackrc }} && openstack overcloud container image prepare
-        --namespace docker.io/tripleo{{ os_version }}
-        --tag {{ container_tag }}
-        --push-destination {{ undercloud_ip }}:8787
-        -e /usr/share/openstack-tripleo-heat-templates/environments/docker.yaml
-        -e /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml
-        --output-images-file overcloud_containers.yml
+        sudo openstack tripleo container image prepare
+        -e /home/stack/containers-prepare-parameter.yaml
         --output-env-file docker-images.yaml
-      become: yes
-      become_user: stack
-    - name: Prepare SDN docker registry image file
-      shell: >
-        {{ stackrc }} && openstack overcloud container image prepare
-        --namespace docker.io/tripleo{{ os_version }}
-        --tag {{ container_tag }}
-        --push-destination {{ undercloud_ip }}:8787
-        {{ sdn_env_file }}
-        --output-images-file sdn_containers.yml
-        --output-env-file sdn-images.yaml
-      become: yes
-      become_user: stack
-      when: sdn != false
-    - name: Update Ceph tag for aarch64 in container env file
-      lineinfile:
-        path: /home/stack/overcloud_containers.yml
-        regexp: '.*ceph.*'
-        line: '- imagename: docker.io/ceph/daemon:master-fafda7d-luminous-centos-7-aarch64'
-      when: aarch64
-    - name: Update Ceph tag for aarch64 in container image file
-      lineinfile:
-        path: /home/stack/docker-images.yaml
-        regexp: '^DockerCephDaemonImage'
-        line: 'DockerCephDaemonImage: {{ undercloud_ip }}:8787/ceph/daemon/master-fafda7d-luminous-centos-7-aarch64'
-      when: aarch64
-    - name: Upload docker images to local registry
-      shell: >
-        {{ stackrc }} && openstack overcloud container image upload
-        --config-file /home/stack/overcloud_containers.yml
-    - name: Upload SDN docker images to local registry
-      shell: >
-        {{ stackrc }} && openstack overcloud container image upload
-        --config-file /home/stack/sdn_containers.yml
-      when: sdn != false
     - name: Collect docker images in registry
       uri:
         url: http://{{ undercloud_ip }}:8787/v2/_catalog
@@ -84,9 +44,8 @@
       with_items: "{{ patched_docker_services }}"
     - name: Modify Images with Apex tag
       replace:
-        path: "{{ item[0] }}"
+        path: "/home/stack/docker-images.yaml"
         regexp: "(\\s*Docker.*?:.*?centos-binary-{{ item[1] }}):.*"
         replace: '\1:apex'
-      with_nested:
-        - [ '/home/stack/sdn-images.yaml', '/home/stack/docker-images.yaml']
-        - "{{ patched_docker_services }}"
\ No newline at end of file
+      with_items: "{{ patched_docker_services }}"
+      become: yes
diff --git a/setup.cfg b/setup.cfg
index 19657b5d..4bb9312e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -32,6 +32,7 @@ packages =
     apex
 data_files =
     share/opnfv-apex/ =
+        build/containers-prepare-parameter.yaml
         build/network-environment.yaml
         build/opnfv-environment.yaml
         build/upstream-environment.yaml