summaryrefslogtreecommitdiffstats
path: root/apex/builders
diff options
context:
space:
mode:
authorTim Rozet <trozet@redhat.com>2017-12-04 11:20:23 -0500
committerTim Rozet <trozet@redhat.com>2018-03-16 14:51:33 -0400
commitf6dbb3929d904b4d5a9ee01f8270051e29ac1ec3 (patch)
treef2490665c2febe0ebc463714f5375483bfca9710 /apex/builders
parenta008f8394e07f1b82d5bf7288f46c63252f6084f (diff)
Enables containerized overcloud deployments
Changes Include: - For upstream deployments, Docker local registry will be updated with latest current RDO containers, regular deployments will use latest stable - Upstream container images will then be patched/modified and then re-uploaded into local docker registry with 'apex' tag - Deployment command modified to deploy with containers - Adds a --no-fetch deployment argument to disable pulling latest from upstream, and instead using what already exists in cache - Moves Undercloud NAT setup to just after undercloud is installed. This provides internet during overcloud install which is now required for upstream container deployments. - Creates loop device for Ceph deployment when no device is provided in deploy settings (for container deployment only) - Updates NIC J2 template to use the new format in OOO since the os-apply-config method is now deprecated in > Queens JIRA: APEX-566 JIRA: APEX-549 Change-Id: I0652c194c059b915a942ac7401936e8f5c69d1fa Signed-off-by: Tim Rozet <trozet@redhat.com>
Diffstat (limited to 'apex/builders')
-rw-r--r--apex/builders/common_builder.py90
-rw-r--r--apex/builders/exceptions.py12
-rw-r--r--apex/builders/overcloud_builder.py73
-rw-r--r--apex/builders/undercloud_builder.py7
4 files changed, 166 insertions, 16 deletions
diff --git a/apex/builders/common_builder.py b/apex/builders/common_builder.py
index fd3bcc3..05a81ef 100644
--- a/apex/builders/common_builder.py
+++ b/apex/builders/common_builder.py
@@ -10,11 +10,16 @@
# Common building utilities for undercloud and overcloud
import git
+import json
import logging
import os
+import re
+import apex.builders.overcloud_builder as oc_builder
from apex import build_utils
+from apex.builders import exceptions as exc
from apex.common import constants as con
+from apex.common import utils
from apex.virtual import utils as virt_utils
@@ -35,9 +40,38 @@ def project_to_path(project):
return "/usr/lib/python2.7/site-packages/{}".format(project)
+def project_to_docker_image(project):
+ """
+ Translates OpenStack project to OOO services that are containerized
+ :param project: name of OpenStack project
+ :return: List of OOO docker service names
+ """
+ # Fetch all docker containers in docker hub with tripleo and filter
+ # based on project
+ hub_output = utils.open_webpage(con.DOCKERHUB_OOO, timeout=10)
+ try:
+ results = json.loads(hub_output.decode())['results']
+ except Exception as e:
+ logging.error("Unable to parse docker hub output for"
+ "tripleoupstream repository")
+ logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
+ raise exc.ApexCommonBuilderException(
+ "Failed to parse docker image info from Docker Hub: {}".format(e))
+ logging.debug("Docker Hub tripleoupstream entities found: {}".format(
+ results))
+ docker_images = list()
+ for result in results:
+ if result['name'].startswith("centos-binary-{}".format(project)):
+ # add as docker image shortname (just service name)
+ docker_images.append(result['name'].replace('centos-binary-', ''))
+
+ return docker_images
+
+
def add_upstream_patches(patches, image, tmp_dir,
default_branch=os.path.join('stable',
- con.DEFAULT_OS_VERSION)):
+ con.DEFAULT_OS_VERSION),
+ uc_ip=None, docker_tag=None):
"""
Adds patches from upstream OpenStack gerrit to Undercloud for deployment
:param patches: list of patches
@@ -45,10 +79,13 @@ def add_upstream_patches(patches, image, tmp_dir,
:param tmp_dir: to store temporary patch files
:param default_branch: default branch to fetch commit (if not specified
in patch)
- :return: None
+ :param uc_ip: undercloud IP (required only for docker patches)
+ :param docker_tag: Docker Tag (required only for docker patches)
+ :return: Set of docker services patched (if applicable)
"""
virt_ops = [{con.VIRT_INSTALL: 'patch'}]
logging.debug("Evaluating upstream patches:\n{}".format(patches))
+ docker_services = set()
for patch in patches:
assert isinstance(patch, dict)
assert all(i in patch.keys() for i in ['project', 'change-id'])
@@ -60,21 +97,52 @@ def add_upstream_patches(patches, image, tmp_dir,
patch['project'], branch)
if patch_diff:
patch_file = "{}.patch".format(patch['change-id'])
- patch_file_path = os.path.join(tmp_dir, patch_file)
+ project_path = project_to_path(patch['project'])
+ # If docker tag and python we know this patch belongs on docker
+ # container for a docker service. Therefore we build the dockerfile
+ # and move the patch into the containers directory. We also assume
+ # this builder call is for overcloud, because we do not support
+ # undercloud containers
+ if docker_tag and 'python' in project_path:
+ # Projects map to multiple THT services, need to check which
+ # are supported
+ ooo_docker_services = project_to_docker_image(patch['project'])
+ else:
+ ooo_docker_services = []
+ # If we found services, then we treat the patch like it applies to
+ # docker only
+ if ooo_docker_services:
+ os_version = default_branch.replace('stable/', '')
+ for service in ooo_docker_services:
+ docker_services = docker_services.union({service})
+ docker_cmds = [
+ "WORKDIR {}".format(project_path),
+ "ADD {} {}".format(patch_file, project_path),
+ "RUN patch -p1 < {}".format(patch_file)
+ ]
+ src_img_uri = "{}:8787/{}/centos-binary-{}:" \
+ "{}".format(uc_ip, os_version, service,
+ docker_tag)
+ oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
+ src_img_uri)
+ patch_file_path = os.path.join(tmp_dir, 'containers',
+ patch_file)
+ else:
+ patch_file_path = os.path.join(tmp_dir, patch_file)
+ virt_ops.extend([
+ {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
+ project_path)},
+ {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
+ project_path, patch_file)}])
+ logging.info("Adding patch {} to {}".format(patch_file,
+ image))
with open(patch_file_path, 'w') as fh:
fh.write(patch_diff)
- project_path = project_to_path(patch['project'])
- virt_ops.extend([
- {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
- project_path)},
- {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
- project_path, patch_file)}])
- logging.info("Adding patch {} to {}".format(patch_file,
- image))
else:
logging.info("Ignoring patch:\n{}".format(patch))
if len(virt_ops) > 1:
virt_utils.virt_customize(virt_ops, image)
+ return docker_services
def add_repo(repo_url, repo_name, image, tmp_dir):
diff --git a/apex/builders/exceptions.py b/apex/builders/exceptions.py
new file mode 100644
index 0000000..b88f02b
--- /dev/null
+++ b/apex/builders/exceptions.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class ApexCommonBuilderException(Exception):
+ pass
diff --git a/apex/builders/overcloud_builder.py b/apex/builders/overcloud_builder.py
index e7b0796..a84d100 100644
--- a/apex/builders/overcloud_builder.py
+++ b/apex/builders/overcloud_builder.py
@@ -10,13 +10,17 @@
# Used to modify overcloud qcow2 image
import logging
+import os
+import tarfile
-from apex.builders import common_builder as c_builder
+import apex.builders.common_builder
from apex.common import constants as con
+from apex.common.exceptions import ApexBuildException
from apex.virtual import utils as virt_utils
-def inject_opendaylight(odl_version, image, tmp_dir):
+def inject_opendaylight(odl_version, image, tmp_dir, uc_ip,
+ os_version, docker_tag=None):
assert odl_version in con.VALID_ODL_VERSIONS
# add repo
if odl_version == 'master':
@@ -28,18 +32,77 @@ def inject_opendaylight(odl_version, image, tmp_dir):
odl_url = "https://nexus.opendaylight.org/content/repositories" \
"/opendaylight-{}-epel-7-x86_64-devel/".format(odl_pkg_version)
repo_name = "opendaylight-{}".format(odl_pkg_version)
- c_builder.add_repo(odl_url, repo_name, image, tmp_dir)
+ apex.builders.common_builder.add_repo(odl_url, repo_name, image, tmp_dir)
# download puppet-opendaylight
- archive = c_builder.create_git_archive(
+ archive = apex.builders.common_builder.create_git_archive(
repo_url=con.PUPPET_ODL_URL, repo_name='puppet-opendaylight',
tmp_dir=tmp_dir, branch=branch, prefix='opendaylight/')
# install ODL, puppet-odl
virt_ops = [
- {con.VIRT_INSTALL: 'opendaylight'},
{con.VIRT_UPLOAD: "{}:/etc/puppet/modules/".format(archive)},
{con.VIRT_RUN_CMD: 'rm -rf /etc/puppet/modules/opendaylight'},
{con.VIRT_RUN_CMD: "cd /etc/puppet/modules/ && tar xvf "
"puppet-opendaylight.tar"}
]
+ if docker_tag:
+ docker_cmds = [
+ "RUN yum remove opendaylight -y",
+ "RUN echo $'[opendaylight]\\n\\",
+ "baseurl={}\\n\\".format(odl_url),
+ "gpgcheck=0\\n\\",
+ "enabled=1' > /etc/yum.repos.d/opendaylight.repo",
+ "RUN yum -y install opendaylight"
+ ]
+ src_img_uri = "{}:8787/{}/centos-binary-{}:" \
+ "{}".format(uc_ip, os_version, 'opendaylight',
+ docker_tag)
+ build_dockerfile('opendaylight', tmp_dir, docker_cmds, src_img_uri)
+ else:
+ virt_ops.append({con.VIRT_INSTALL: 'opendaylight'})
virt_utils.virt_customize(virt_ops, image)
logging.info("OpenDaylight injected into {}".format(image))
+
+
+def build_dockerfile(service, tmp_dir, docker_cmds, src_image_uri):
+ """
+ Builds docker file per service and stores it in a
+ tmp_dir/containers/<service> directory. If the Dockerfile already exists,
+ simply append the docker cmds to it.
+ :param service: name of sub-directory to store Dockerfile in
+ :param tmp_dir: Temporary directory to store the container's dockerfile in
+ :param docker_cmds: List of commands to insert into the dockerfile
+ :param src_image_uri: Docker URI format for where the source image exists
+ :return: None
+ """
+ logging.debug("Building Dockerfile for {} with docker_cmds: {}".format(
+ service, docker_cmds))
+ c_dir = os.path.join(tmp_dir, 'containers')
+ service_dir = os.path.join(c_dir, service)
+ if not os.path.isdir(service_dir):
+ os.makedirs(service_dir, exist_ok=True)
+ from_cmd = "FROM {}\n".format(src_image_uri)
+ service_file = os.path.join(service_dir, 'Dockerfile')
+ assert isinstance(docker_cmds, list)
+ if os.path.isfile(service_file):
+ append_cmds = True
+ else:
+ append_cmds = False
+ with open(service_file, "a+") as fh:
+ if not append_cmds:
+ fh.write(from_cmd)
+ fh.write('\n'.join(docker_cmds))
+
+
+def archive_docker_patches(tmp_dir):
+ """
+ Archives Overcloud docker patches into a tar file for upload to Undercloud
+ :param tmp_dir: temporary directory where containers folder is stored
+ :return: None
+ """
+ container_path = os.path.join(tmp_dir, 'containers')
+ if not os.path.isdir(container_path):
+ raise ApexBuildException("Docker directory for patches not found: "
+ "{}".format(container_path))
+ archive_file = os.path.join(tmp_dir, 'docker_patches.tar.gz')
+ with tarfile.open(archive_file, "w:gz") as tar:
+ tar.add(container_path, arcname=os.path.basename(container_path))
diff --git a/apex/builders/undercloud_builder.py b/apex/builders/undercloud_builder.py
index baba8a5..268bad7 100644
--- a/apex/builders/undercloud_builder.py
+++ b/apex/builders/undercloud_builder.py
@@ -20,6 +20,11 @@ def add_upstream_packages(image):
:return: None
"""
virt_ops = list()
+ # FIXME(trozet): we have to lock to this beta ceph ansible package because
+ # the current RPM versioning is wrong and an older package has a higher
+ # version than this package. We should change to just 'ceph-ansible'
+ # once the package/repo has been fixed. Note: luminous is fine here
+ # because Apex will only support container deployment for Queens and later
pkgs = [
'openstack-utils',
'ceph-common',
@@ -29,6 +34,8 @@ def add_upstream_packages(image):
'docker-distribution',
'openstack-tripleo-validations',
'libguestfs-tools',
+ 'http://mirror.centos.org/centos/7/storage/x86_64/ceph-luminous' +
+ '/ceph-ansible-3.1.0-0.beta3.1.el7.noarch.rpm'
]
for pkg in pkgs: