From d38e8a91c1a0e3f94d2a19876956ade48d1a8ed2 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Fri, 26 Oct 2018 14:03:17 -0400 Subject: Fixes Docker image upload for master/rocky The API has changed to create/upload the docker container images to be used for deployment. In the past the prepare commands would read the THT environment files passed, to determine which docker images to render into an environment file. The new behavior uses a new "containers-prepare-parameter.yaml" format (included in this patch), which Apex will now configure for deployment. By default docker images will be rendered for all TripleO services identified in the roles_data.yaml file. Therefore we must use several excludes patterns to only pull the docker images needed for a default deployment. JIRA: APEX-642 Change-Id: Iab00fcb874554bb98540dc9a4c3051e58ea68a3b Signed-off-by: Tim Rozet --- apex/builders/common_builder.py | 39 ++++++++++++++++++++++++ apex/deploy.py | 9 ++++++ apex/overcloud/deploy.py | 2 -- apex/tests/test_apex_common_builder.py | 52 ++++++++++++++++++++++++++++++++ apex/tests/test_apex_deploy.py | 10 ++++-- apex/tests/test_apex_overcloud_deploy.py | 1 - apex/undercloud/undercloud.py | 4 ++- 7 files changed, 110 insertions(+), 7 deletions(-) (limited to 'apex') diff --git a/apex/builders/common_builder.py b/apex/builders/common_builder.py index b8894ec1..e0e7303e 100644 --- a/apex/builders/common_builder.py +++ b/apex/builders/common_builder.py @@ -14,8 +14,11 @@ import git import json import logging import os +import platform +import pprint import re import urllib.parse +import yaml import apex.builders.overcloud_builder as oc_builder from apex import build_utils @@ -258,3 +261,39 @@ def create_git_archive(repo_url, repo_name, tmp_dir, repo.archive(fh, prefix=prefix) logging.debug("Wrote archive file: {}".format(archive_path)) return archive_path + + +def get_neutron_driver(ds_opts): + sdn = ds_opts.get('sdn_controller', None) + for controllers in 'opendaylight', 'ovn': + if sdn == controllers: + return sdn + + if ds_opts.get('vpp', False): + return 'vpp' + + return None + + +def prepare_container_images(prep_file, branch='master', neutron_driver=None): + if not os.path.isfile(prep_file): + raise exc.ApexCommonBuilderException("Prep file does not exist: " + "{}".format(prep_file)) + with open(prep_file) as fh: + data = yaml.safe_load(fh) + try: + p_set = data['parameter_defaults']['ContainerImagePrepare'][0]['set'] + if neutron_driver: + p_set['neutron_driver'] = neutron_driver + p_set['namespace'] = "docker.io/tripleo{}".format(branch) + if platform.machine() == 'aarch64': + p_set['ceph_tag'] = 'master-fafda7d-luminous-centos-7-aarch64' + + except KeyError: + logging.error("Invalid prep file format: {}".format(prep_file)) + raise exc.ApexCommonBuilderException("Invalid format for prep file") + + logging.debug("Writing new container prep file:\n{}".format( + pprint.pformat(data))) + with open(prep_file, 'w') as fh: + yaml.safe_dump(data, fh, default_flow_style=False) diff --git a/apex/deploy.py b/apex/deploy.py index dab6bd1e..f1ab260f 100644 --- a/apex/deploy.py +++ b/apex/deploy.py @@ -415,6 +415,15 @@ def main(): for role in 'compute', 'controller': oc_cfg.create_nic_template(net_settings, deploy_settings, role, args.deploy_dir, APEX_TEMP_DIR) + # Prepare/Upload docker images + docker_env = 'containers-prepare-parameter.yaml' + shutil.copyfile(os.path.join(args.deploy_dir, docker_env), + os.path.join(APEX_TEMP_DIR, docker_env)) + c_builder.prepare_container_images( + os.path.join(APEX_TEMP_DIR, docker_env), + branch=branch, + neutron_driver=c_builder.get_neutron_driver(ds_opts) + ) # Install Undercloud undercloud.configure(net_settings, deploy_settings, os.path.join(args.lib_dir, constants.ANSIBLE_PATH, diff --git a/apex/overcloud/deploy.py b/apex/overcloud/deploy.py index 4deeabff..c526a98e 100644 --- a/apex/overcloud/deploy.py +++ b/apex/overcloud/deploy.py @@ -194,8 +194,6 @@ def create_deploy_cmd(ds, ns, inv, tmp_dir, sdn_docker_files = get_docker_sdn_files(ds_opts) for sdn_docker_file in sdn_docker_files: deploy_options.append(sdn_docker_file) - if sdn_docker_files: - deploy_options.append('sdn-images.yaml') else: deploy_options += build_sdn_env_list(ds_opts, SDN_FILE_MAP) diff --git a/apex/tests/test_apex_common_builder.py b/apex/tests/test_apex_common_builder.py index 09bd2545..dede55a2 100644 --- a/apex/tests/test_apex_common_builder.py +++ b/apex/tests/test_apex_common_builder.py @@ -24,6 +24,8 @@ DOCKER_YAML = { } } +a_mock_open = mock_open(read_data=None) + class TestCommonBuilder(unittest.TestCase): @classmethod @@ -249,3 +251,53 @@ class TestCommonBuilder(unittest.TestCase): self.assertRaises(exceptions.ApexCommonBuilderException, c_builder.project_to_docker_image, 'nova') + + def test_get_neutron_driver(self): + ds_opts = {'dataplane': 'fdio', + 'sdn_controller': 'opendaylight', + 'odl_version': 'master', + 'vpn': False, + 'sriov': False} + self.assertEquals(c_builder.get_neutron_driver(ds_opts), + 'opendaylight') + ds_opts['sdn_controller'] = None + ds_opts['vpp'] = True + self.assertEquals(c_builder.get_neutron_driver(ds_opts), + 'vpp') + ds_opts['sdn_controller'] = 'ovn' + self.assertEquals(c_builder.get_neutron_driver(ds_opts), + 'ovn') + + @patch('apex.builders.common_builder.yaml') + @patch('apex.overcloud.deploy.os.path.isfile') + @patch('builtins.open', a_mock_open, create=True) + def test_prepare_container_images(self, mock_is_file, mock_yaml): + mock_yaml.safe_load.return_value = { + 'parameter_defaults': { + 'ContainerImagePrepare': [ + {'set': + {'namespace': 'blah', + 'neutron_driver': 'null', + } + } + ] + } + } + expected_output = { + 'parameter_defaults': { + 'ContainerImagePrepare': [ + {'set': + {'namespace': 'docker.io/tripleoqueens', + 'neutron_driver': 'opendaylight', + } + } + ] + } + } + + c_builder.prepare_container_images('dummy.yaml', 'queens', + 'opendaylight') + mock_yaml.safe_dump.assert_called_with( + expected_output, + a_mock_open.return_value, + default_flow_style=False) diff --git a/apex/tests/test_apex_deploy.py b/apex/tests/test_apex_deploy.py index be52c276..004c21c1 100644 --- a/apex/tests/test_apex_deploy.py +++ b/apex/tests/test_apex_deploy.py @@ -118,6 +118,7 @@ class TestDeploy(unittest.TestCase): args.snapshot = False assert_raises(ApexDeployException, validate_deploy_args, args) + @patch('apex.deploy.c_builder') @patch('apex.deploy.ApexDeployment') @patch('apex.deploy.uc_builder') @patch('apex.deploy.network_data.create_network_data') @@ -146,7 +147,7 @@ class TestDeploy(unittest.TestCase): mock_utils, mock_parsers, mock_oc_cfg, mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib, mock_oc_deploy, mock_shutil, mock_network_data, - mock_uc_builder, mock_deployment): + mock_uc_builder, mock_deployment, mock_c_builder): net_sets_dict = {'networks': MagicMock(), 'dns_servers': 'test'} ds_opts_dict = {'global_params': MagicMock(), @@ -197,6 +198,7 @@ class TestDeploy(unittest.TestCase): main() mock_snap_deployment.assert_called() + @patch('apex.deploy.c_builder') @patch('apex.deploy.ApexDeployment') @patch('apex.deploy.uc_builder') @patch('apex.deploy.network_data.create_network_data') @@ -225,7 +227,7 @@ class TestDeploy(unittest.TestCase): mock_utils, mock_parsers, mock_oc_cfg, mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib, mock_oc_deploy, mock_shutil, mock_network_data, - mock_uc_builder, mock_deployment): + mock_uc_builder, mock_deployment, mock_c_builder): # didn't work yet line 412 # net_sets_dict = {'networks': {'admin': {'cidr': MagicMock()}}, # 'dns_servers': 'test'} @@ -329,6 +331,7 @@ class TestDeploy(unittest.TestCase): # TODO(trozet) add assertions here with arguments for functions in # deploy main + @patch('apex.deploy.c_builder') @patch('apex.deploy.ApexDeployment') @patch('apex.deploy.uc_builder') @patch('apex.deploy.network_data.create_network_data') @@ -358,7 +361,8 @@ class TestDeploy(unittest.TestCase): mock_utils, mock_parsers, mock_oc_cfg, mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib, mock_oc_deploy, mock_git, mock_shutil, - mock_network_data, mock_uc_builder, mock_deployment): + mock_network_data, mock_uc_builder, mock_deployment, + mock_c_builder): net_sets_dict = {'networks': MagicMock(), 'dns_servers': 'test'} ds_opts_dict = {'global_params': MagicMock(), diff --git a/apex/tests/test_apex_overcloud_deploy.py b/apex/tests/test_apex_overcloud_deploy.py index a70057b9..402ecebf 100644 --- a/apex/tests/test_apex_overcloud_deploy.py +++ b/apex/tests/test_apex_overcloud_deploy.py @@ -156,7 +156,6 @@ class TestOvercloudDeploy(unittest.TestCase): assert_in('--control-scale 3', result_cmd) assert_in('--compute-scale 2', result_cmd) assert_in('docker-images.yaml', result_cmd) - assert_in('sdn-images.yaml', result_cmd) assert_in('/usr/share/openstack-tripleo-heat-templates/environments' '/docker.yaml', result_cmd) assert_in('/usr/share/openstack-tripleo-heat-templates/environments/' diff --git a/apex/undercloud/undercloud.py b/apex/undercloud/undercloud.py index 1658801d..8b6b9d4c 100644 --- a/apex/undercloud/undercloud.py +++ b/apex/undercloud/undercloud.py @@ -235,7 +235,9 @@ class Undercloud: "local_ip {}/{}".format(str(ns_admin['installer_vm']['ip']), str(ns_admin['cidr']).split('/')[1]), "generate_service_certificate false", - "undercloud_ntp_servers {}".format(str(ns['ntp'][0])) + "undercloud_ntp_servers {}".format(str(ns['ntp'][0])), + "container_images_file " + "/home/stack/containers-prepare-parameter.yaml" ] config['undercloud_network_config'] = [ -- cgit 1.2.3-korg