From c4b66d46770847e9090a06c4c2ec3e8266606282 Mon Sep 17 00:00:00 2001 From: Ricardo Noriega Date: Fri, 27 Jul 2018 11:29:41 +0200 Subject: Enable BGPVPN for master deployments - Injection of Quagga tarball via overcloud builder. - Extraction and installation of all related packages. - It uses SDNVPN artifact repository to download Quagga tarball, so there is only one source to test. - Modifies bgpvpn scenario files to use OS master branch, ODL master branch and containers. JIRA: APEX-627 Change-Id: Icdbc2853d9531048e23fd6d5e444bd68208d18fc Signed-off-by: Ricardo Noriega --- apex/builders/overcloud_builder.py | 23 +++++++++++ apex/common/constants.py | 3 ++ apex/deploy.py | 7 +++- apex/overcloud/deploy.py | 30 +++++++------- apex/tests/test_apex_overcloud_deploy.py | 48 ++++++++++++++++++---- build/rpm_specs/opnfv-apex.spec | 2 + config/deploy/os-odl-bgpvpn-ha.yaml | 6 ++- config/deploy/os-odl-bgpvpn-noha.yaml | 6 ++- config/deploy/os-odl-bgpvpn_queens-ha.yaml | 13 ++++++ config/deploy/os-odl-bgpvpn_queens-noha.yaml | 13 ++++++ .../playbooks/prepare_overcloud_containers.yml | 2 +- 11 files changed, 124 insertions(+), 29 deletions(-) create mode 100644 config/deploy/os-odl-bgpvpn_queens-ha.yaml create mode 100644 config/deploy/os-odl-bgpvpn_queens-noha.yaml diff --git a/apex/builders/overcloud_builder.py b/apex/builders/overcloud_builder.py index d6f03eb2..b8556456 100644 --- a/apex/builders/overcloud_builder.py +++ b/apex/builders/overcloud_builder.py @@ -15,6 +15,7 @@ import tarfile import apex.builders.common_builder from apex.common import constants as con +from apex.common import utils as utils from apex.common.exceptions import ApexBuildException from apex.virtual import utils as virt_utils @@ -64,6 +65,28 @@ def inject_opendaylight(odl_version, image, tmp_dir, uc_ip, logging.info("OpenDaylight injected into {}".format(image)) +def inject_quagga(image, tmp_dir): + """ + Downloads quagga tarball from artifacts.opnfv.org + and install it on the overcloud image on the fly. + :param image: + :param tmp_dir: + :return: + """ + utils.fetch_upstream_and_unpack(tmp_dir, + os.path.split(con.QUAGGA_URL)[0] + "/", + [os.path.basename(con.QUAGGA_URL)]) + + virt_ops = [ + {con.VIRT_UPLOAD: "{}/quagga-4.tar.gz:/root/".format(tmp_dir)}, + {con.VIRT_RUN_CMD: "cd /root/ && tar xzf quagga-4.tar.gz"}, + {con.VIRT_RUN_CMD: "cd /root/quagga;packages=$(ls |grep -vE 'debug" + "info|devel|contrib');yum -y install $packages"} + ] + virt_utils.virt_customize(virt_ops, image) + logging.info("Quagga injected into {}".format(image)) + + def build_dockerfile(service, tmp_dir, docker_cmds, src_image_uri): """ Builds docker file per service and stores it in a diff --git a/apex/common/constants.py b/apex/common/constants.py index d855222c..138166f0 100644 --- a/apex/common/constants.py +++ b/apex/common/constants.py @@ -63,6 +63,7 @@ VALID_DOCKER_SERVICES = { 'neutron-opendaylight.yaml': None, 'neutron-opendaylight-dpdk.yaml': None, 'neutron-opendaylight-sriov.yaml': None, + 'neutron-bgpvpn-opendaylight.yaml': None, 'neutron-ml2-ovn.yaml': 'neutron-ovn.yaml' } DOCKERHUB_OOO = 'https://registry.hub.docker.com/v2/repositories' \ @@ -70,3 +71,5 @@ DOCKERHUB_OOO = 'https://registry.hub.docker.com/v2/repositories' \ KUBESPRAY_URL = 'https://github.com/kubernetes-incubator/kubespray.git' CUSTOM_OVS = 'http://artifacts.opnfv.org/apex/random/openvswitch-2.9.0-9' \ '.el7fdn.x86_64.rpm' + +QUAGGA_URL = "http://artifacts.opnfv.org/sdnvpn/quagga/quagga-4.tar.gz" diff --git a/apex/deploy.py b/apex/deploy.py index 65f00e04..70bc3a5a 100644 --- a/apex/deploy.py +++ b/apex/deploy.py @@ -433,6 +433,10 @@ def main(): 'prepare_overcloud_containers.yml') if ds_opts['containers']: logging.info("Preparing Undercloud with Docker containers") + sdn_env = oc_deploy.get_docker_sdn_files(ds_opts) + sdn_env_files = str() + for sdn_file in sdn_env: + sdn_env_files += " -e {}".format(sdn_file) if patched_containers: oc_builder.archive_docker_patches(APEX_TEMP_DIR) container_vars = dict() @@ -445,8 +449,7 @@ def main(): container_vars['undercloud_ip'] = undercloud_admin_ip container_vars['os_version'] = os_version container_vars['aarch64'] = platform.machine() == 'aarch64' - container_vars['sdn_env_file'] = \ - oc_deploy.get_docker_sdn_file(ds_opts) + container_vars['sdn_env_file'] = sdn_env_files try: utils.run_ansible(container_vars, docker_playbook, host=undercloud.ip, user='stack', diff --git a/apex/overcloud/deploy.py b/apex/overcloud/deploy.py index 716c57c3..790e794a 100644 --- a/apex/overcloud/deploy.py +++ b/apex/overcloud/deploy.py @@ -138,24 +138,24 @@ def build_sdn_env_list(ds, sdn_map, env_list=None): return env_list -def get_docker_sdn_file(ds_opts): +def get_docker_sdn_files(ds_opts): """ Returns docker env file for detected SDN :param ds_opts: deploy options - :return: docker THT env file for an SDN + :return: list of docker THT env files for an SDN """ - # FIXME(trozet): We assume right now there is only one docker SDN file docker_services = con.VALID_DOCKER_SERVICES tht_dir = con.THT_DOCKER_ENV_DIR[ds_opts['os_version']] sdn_env_list = build_sdn_env_list(ds_opts, SDN_FILE_MAP) - for sdn_file in sdn_env_list: + for i, sdn_file in enumerate(sdn_env_list): sdn_base = os.path.basename(sdn_file) if sdn_base in docker_services: if docker_services[sdn_base] is not None: - return os.path.join(tht_dir, - docker_services[sdn_base]) + sdn_env_list[i] = \ + os.path.join(tht_dir, docker_services[sdn_base]) else: - return os.path.join(tht_dir, sdn_base) + sdn_env_list[i] = os.path.join(tht_dir, sdn_base) + return sdn_env_list def create_deploy_cmd(ds, ns, inv, tmp_dir, @@ -184,9 +184,10 @@ def create_deploy_cmd(ds, ns, inv, tmp_dir, if ds_opts['containers']: deploy_options.append('docker-images.yaml') - sdn_docker_file = get_docker_sdn_file(ds_opts) - if sdn_docker_file: + sdn_docker_files = get_docker_sdn_files(ds_opts) + for sdn_docker_file in sdn_docker_files: deploy_options.append(sdn_docker_file) + if sdn_docker_files: deploy_options.append('sdn-images.yaml') else: deploy_options += build_sdn_env_list(ds_opts, SDN_FILE_MAP) @@ -306,7 +307,13 @@ def prep_image(ds, ns, img, tmp_dir, root_pw=None, docker_tag=None, "echo 'https_proxy={}' >> /etc/environment".format( ns['https_proxy'])}) + tmp_oc_image = os.path.join(tmp_dir, 'overcloud-full.qcow2') + shutil.copyfile(img, tmp_oc_image) + logging.debug("Temporary overcloud image stored as: {}".format( + tmp_oc_image)) + if ds_opts['vpn']: + oc_builder.inject_quagga(tmp_oc_image, tmp_dir) virt_cmds.append({con.VIRT_RUN_CMD: "chmod +x /etc/rc.d/rc.local"}) virt_cmds.append({ con.VIRT_RUN_CMD: @@ -383,11 +390,6 @@ def prep_image(ds, ns, img, tmp_dir, root_pw=None, docker_tag=None, "/root/nosdn_vpp_rpms/*.rpm"} ]) - tmp_oc_image = os.path.join(tmp_dir, 'overcloud-full.qcow2') - shutil.copyfile(img, tmp_oc_image) - logging.debug("Temporary overcloud image stored as: {}".format( - tmp_oc_image)) - if sdn == 'opendaylight': undercloud_admin_ip = ns['networks'][con.ADMIN_NETWORK][ 'installer_vm']['ip'] diff --git a/apex/tests/test_apex_overcloud_deploy.py b/apex/tests/test_apex_overcloud_deploy.py index 17911d59..41f2e01a 100644 --- a/apex/tests/test_apex_overcloud_deploy.py +++ b/apex/tests/test_apex_overcloud_deploy.py @@ -29,7 +29,7 @@ from apex.overcloud.deploy import prep_sriov_env from apex.overcloud.deploy import external_network_cmds from apex.overcloud.deploy import create_congress_cmds from apex.overcloud.deploy import SDN_FILE_MAP -from apex.overcloud.deploy import get_docker_sdn_file +from apex.overcloud.deploy import get_docker_sdn_files from nose.tools import ( assert_regexp_matches, @@ -133,6 +133,7 @@ class TestOvercloudDeploy(unittest.TestCase): 'tacker': False, 'containers': True, 'barometer': False, + 'vpn': False, 'ceph': True, 'sdn_controller': 'opendaylight', 'sriov': False, @@ -219,6 +220,7 @@ class TestOvercloudDeploy(unittest.TestCase): ds_opts = {'dataplane': 'fdio', 'sdn_controller': 'opendaylight', 'odl_version': 'master', + 'vpn': False, 'sriov': False} ds = {'deploy_options': MagicMock(), 'global_params': MagicMock()} @@ -236,6 +238,7 @@ class TestOvercloudDeploy(unittest.TestCase): def test_prep_image_sdn_false(self, mock_os_path, mock_shutil, mock_virt_utils): ds_opts = {'dataplane': 'fdio', + 'vpn': False, 'sdn_controller': False} ds = {'deploy_options': MagicMock(), 'global_params': MagicMock()} @@ -254,6 +257,7 @@ class TestOvercloudDeploy(unittest.TestCase): mock_virt_utils, mock_inject_odl): ds_opts = {'dataplane': 'ovs', 'sdn_controller': 'opendaylight', + 'vpn': False, 'odl_version': con.DEFAULT_ODL_VERSION, 'odl_vpp_netvirt': True} ds = {'deploy_options': MagicMock(), @@ -323,6 +327,7 @@ class TestOvercloudDeploy(unittest.TestCase): def test_prep_image_sdn_ovn(self, mock_os_path, mock_shutil, mock_virt_utils): ds_opts = {'dataplane': 'ovs', + 'vpn': False, 'sdn_controller': 'ovn'} ds = {'deploy_options': MagicMock(), 'global_params': MagicMock()} @@ -332,6 +337,32 @@ class TestOvercloudDeploy(unittest.TestCase): prep_image(ds, ns, 'undercloud.qcow2', '/tmp', root_pw='test') mock_virt_utils.virt_customize.assert_called() + @patch('apex.builders.overcloud_builder.inject_quagga') + @patch('apex.builders.overcloud_builder.inject_opendaylight') + @patch('apex.overcloud.deploy.virt_utils') + @patch('apex.overcloud.deploy.shutil') + @patch('apex.overcloud.deploy.os.path') + @patch('builtins.open', mock_open()) + def test_prep_image_sdn_odl_vpn(self, mock_os_path, mock_shutil, + mock_virt_utils, mock_inject_odl, + mock_inject_quagga): + ds_opts = {'dataplane': 'ovs', + 'sdn_controller': 'opendaylight', + 'vpn': True, + 'odl_version': con.DEFAULT_ODL_VERSION, + 'odl_vpp_netvirt': True} + ds = {'deploy_options': MagicMock(), + 'global_params': MagicMock()} + ds['deploy_options'].__getitem__.side_effect = \ + lambda i: ds_opts.get(i, MagicMock()) + ds['deploy_options'].__contains__.side_effect = \ + lambda i: True if i in ds_opts else MagicMock() + ns = MagicMock() + prep_image(ds, ns, 'undercloud.qcow2', '/tmp', root_pw='test') + mock_virt_utils.virt_customize.assert_called() + mock_inject_odl.assert_called() + mock_inject_quagga.assert_called() + @patch('apex.overcloud.deploy.os.path.isfile') def test_prep_image_no_image(self, mock_isfile): mock_isfile.return_value = False @@ -767,19 +798,20 @@ class TestOvercloudDeploy(unittest.TestCase): mock_parsers.return_value.__getitem__.side_effect = KeyError() assert_raises(KeyError, create_congress_cmds, 'overcloud_file') - def test_get_docker_sdn_file(self): + def test_get_docker_sdn_files(self): ds_opts = {'ha_enabled': True, 'congress': True, 'tacker': True, 'containers': False, 'barometer': True, 'ceph': False, + 'vpn': True, 'sdn_controller': 'opendaylight', 'os_version': 'queens' } - output = get_docker_sdn_file(ds_opts) - self.assertEqual(output, - ('/usr/share/openstack-tripleo-heat-templates' - '/environments/services/neutron-opendaylight' - '.yaml') - ) + output = get_docker_sdn_files(ds_opts) + compare = ['/usr/share/openstack-tripleo-heat-templates/' + 'environments/services/neutron-opendaylight.yaml', + '/usr/share/openstack-tripleo-heat-templates/environments' + '/services/neutron-bgpvpn-opendaylight.yaml'] + self.assertEqual(output, compare) diff --git a/build/rpm_specs/opnfv-apex.spec b/build/rpm_specs/opnfv-apex.spec index c6bb5308..c344dfda 100644 --- a/build/rpm_specs/opnfv-apex.spec +++ b/build/rpm_specs/opnfv-apex.spec @@ -87,6 +87,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/ %{_sysconfdir}/opnfv-apex/os-nosdn-kvm_ovs_dpdk-noha.yaml %{_sysconfdir}/opnfv-apex/os-odl-bgpvpn-ha.yaml %{_sysconfdir}/opnfv-apex/os-odl-bgpvpn-noha.yaml +%{_sysconfdir}/opnfv-apex/os-odl-bgpvpn_queens-ha.yaml +%{_sysconfdir}/opnfv-apex/os-odl-bgpvpn_queens-noha.yaml %{_sysconfdir}/opnfv-apex/os-odl-sfc-ha.yaml %{_sysconfdir}/opnfv-apex/os-odl-sfc-noha.yaml %{_sysconfdir}/opnfv-apex/os-odl-fdio-noha.yaml diff --git a/config/deploy/os-odl-bgpvpn-ha.yaml b/config/deploy/os-odl-bgpvpn-ha.yaml index 68b62734..f61e1a60 100644 --- a/config/deploy/os-odl-bgpvpn-ha.yaml +++ b/config/deploy/os-odl-bgpvpn-ha.yaml @@ -3,9 +3,11 @@ global_params: ha_enabled: true deploy_options: + containers: true + os_version: master sdn_controller: opendaylight - odl_version: nitrogen + odl_version: master tacker: false - congress: true + congress: false sfc: false vpn: true diff --git a/config/deploy/os-odl-bgpvpn-noha.yaml b/config/deploy/os-odl-bgpvpn-noha.yaml index 0435e6b5..4bf90aa7 100644 --- a/config/deploy/os-odl-bgpvpn-noha.yaml +++ b/config/deploy/os-odl-bgpvpn-noha.yaml @@ -3,9 +3,11 @@ global_params: ha_enabled: false deploy_options: + containers: true + os_version: master sdn_controller: opendaylight - odl_version: nitrogen + odl_version: master tacker: false - congress: true + congress: false sfc: false vpn: true diff --git a/config/deploy/os-odl-bgpvpn_queens-ha.yaml b/config/deploy/os-odl-bgpvpn_queens-ha.yaml new file mode 100644 index 00000000..27a0caf6 --- /dev/null +++ b/config/deploy/os-odl-bgpvpn_queens-ha.yaml @@ -0,0 +1,13 @@ +--- +global_params: + ha_enabled: true + +deploy_options: + containers: true + os_version: queens + sdn_controller: opendaylight + odl_version: master + tacker: false + congress: false + sfc: false + vpn: true diff --git a/config/deploy/os-odl-bgpvpn_queens-noha.yaml b/config/deploy/os-odl-bgpvpn_queens-noha.yaml new file mode 100644 index 00000000..33382307 --- /dev/null +++ b/config/deploy/os-odl-bgpvpn_queens-noha.yaml @@ -0,0 +1,13 @@ +--- +global_params: + ha_enabled: false + +deploy_options: + containers: true + os_version: queens + sdn_controller: opendaylight + odl_version: master + tacker: false + congress: false + sfc: false + vpn: true diff --git a/lib/ansible/playbooks/prepare_overcloud_containers.yml b/lib/ansible/playbooks/prepare_overcloud_containers.yml index 1651a531..54dbe098 100644 --- a/lib/ansible/playbooks/prepare_overcloud_containers.yml +++ b/lib/ansible/playbooks/prepare_overcloud_containers.yml @@ -36,7 +36,7 @@ --namespace docker.io/tripleo{{ os_version }} --tag {{ container_tag }} --push-destination {{ undercloud_ip }}:8787 - -e {{ sdn_env_file }} + {{ sdn_env_file }} --output-images-file sdn_containers.yml --output-env-file sdn-images.yaml become: yes -- cgit 1.2.3-korg