From f6dbb3929d904b4d5a9ee01f8270051e29ac1ec3 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Mon, 4 Dec 2017 11:20:23 -0500 Subject: Enables containerized overcloud deployments Changes Include: - For upstream deployments, Docker local registry will be updated with latest current RDO containers, regular deployments will use latest stable - Upstream container images will then be patched/modified and then re-uploaded into local docker registry with 'apex' tag - Deployment command modified to deploy with containers - Adds a --no-fetch deployment argument to disable pulling latest from upstream, and instead using what already exists in cache - Moves Undercloud NAT setup to just after undercloud is installed. This provides internet during overcloud install which is now required for upstream container deployments. - Creates loop device for Ceph deployment when no device is provided in deploy settings (for container deployment only) - Updates NIC J2 template to use the new format in OOO since the os-apply-config method is now deprecated in > Queens JIRA: APEX-566 JIRA: APEX-549 Change-Id: I0652c194c059b915a942ac7401936e8f5c69d1fa Signed-off-by: Tim Rozet --- apex/common/constants.py | 16 ++++++++++++ apex/common/exceptions.py | 4 +++ apex/common/utils.py | 63 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 72 insertions(+), 11 deletions(-) (limited to 'apex/common') diff --git a/apex/common/constants.py b/apex/common/constants.py index a2b9a634..0aa6a6ca 100644 --- a/apex/common/constants.py +++ b/apex/common/constants.py @@ -39,10 +39,14 @@ VIRT_PW = '--root-password' THT_DIR = '/usr/share/openstack-tripleo-heat-templates' THT_ENV_DIR = os.path.join(THT_DIR, 'environments') +THT_DOCKER_ENV_DIR = os.path.join(THT_ENV_DIR, 'services-docker') DEFAULT_OS_VERSION = 'pike' DEFAULT_ODL_VERSION = 'nitrogen' VALID_ODL_VERSIONS = ['carbon', 'nitrogen', 'oxygen', 'master'] +CEPH_VERSION_MAP = {'pike': 'jewel', + 'queens': 'luminous', + 'master': 'luminous'} PUPPET_ODL_URL = 'https://git.opendaylight.org/gerrit/integration/packaging' \ '/puppet-opendaylight' DEBUG_OVERCLOUD_PW = 'opnfvapex' @@ -50,3 +54,15 @@ NET_ENV_FILE = 'network-environment.yaml' DEPLOY_TIMEOUT = 90 UPSTREAM_RDO = 'https://images.rdoproject.org/pike/delorean/current-tripleo/' OPENSTACK_GERRIT = 'https://review.openstack.org' + +DOCKER_TAG = 'current-tripleo-rdo' +# Maps regular service files to docker versions +# None value means mapping is same as key +VALID_DOCKER_SERVICES = { + 'neutron-opendaylight.yaml': None, + 'neutron-opendaylight-dpdk.yaml': None, + 'neutron-opendaylight-sriov.yaml': None, + 'neutron-ml2-ovn.yaml': 'neutron-ovn.yaml' +} +DOCKERHUB_OOO = ('https://registry.hub.docker.com/v2/repositories' + '/tripleoupstream/?page_size=1024') diff --git a/apex/common/exceptions.py b/apex/common/exceptions.py index 54d99834..a4d390a4 100644 --- a/apex/common/exceptions.py +++ b/apex/common/exceptions.py @@ -18,3 +18,7 @@ class JumpHostNetworkException(Exception): class ApexCleanException(Exception): pass + + +class ApexBuildException(Exception): + pass diff --git a/apex/common/utils.py b/apex/common/utils.py index b727b11a..cb7cbe13 100644 --- a/apex/common/utils.py +++ b/apex/common/utils.py @@ -22,6 +22,8 @@ import urllib.request import urllib.parse import yaml +from apex.common import exceptions as exc + def str2bool(var): if isinstance(var, bool): @@ -139,30 +141,45 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root', raise Exception(e) -def fetch_upstream_and_unpack(dest, url, targets): +def fetch_upstream_and_unpack(dest, url, targets, fetch=True): """ Fetches targets from a url destination and downloads them if they are newer. Also unpacks tar files in dest dir. :param dest: Directory to download and unpack files to :param url: URL where target files are located :param targets: List of target files to download + :param fetch: Whether or not to fetch latest from internet (boolean) :return: None """ os.makedirs(dest, exist_ok=True) assert isinstance(targets, list) for target in targets: - download_target = True target_url = urllib.parse.urljoin(url, target) target_dest = os.path.join(dest, target) - logging.debug("Fetching and comparing upstream target: \n{}".format( - target_url)) - try: - u = urllib.request.urlopen(target_url) - except urllib.error.URLError as e: - logging.error("Failed to fetch target url. Error: {}".format( - e.reason)) - raise - if os.path.isfile(target_dest): + target_exists = os.path.isfile(target_dest) + if fetch: + download_target = True + elif not target_exists: + logging.warning("no-fetch requested but target: {} is not " + "cached, will download".format(target_dest)) + download_target = True + else: + logging.info("no-fetch requested and previous cache exists for " + "target: {}. Will skip download".format(target_dest)) + download_target = False + + if download_target: + logging.debug("Fetching and comparing upstream" + " target: \n{}".format(target_url)) + try: + u = urllib.request.urlopen(target_url) + except urllib.error.URLError as e: + logging.error("Failed to fetch target url. Error: {}".format( + e.reason)) + raise + # Check if previous file and fetch we need to compare files to + # determine if download is necessary + if target_exists and download_target: logging.debug("Previous file found: {}".format(target_dest)) metadata = u.info() headers = metadata.items() @@ -186,6 +203,7 @@ def fetch_upstream_and_unpack(dest, url, targets): download_target = False else: logging.debug('Unable to find last modified url date') + if download_target: urllib.request.urlretrieve(target_url, filename=target_dest) logging.info("Target downloaded: {}".format(target)) @@ -220,3 +238,26 @@ def internet_connectivity(): except (urllib.request.URLError, socket.timeout): logging.debug('No internet connectivity detected') return False + + +def open_webpage(url, timeout=5): + try: + response = urllib.request.urlopen(url, timeout=timeout) + return response.read() + except (urllib.request.URLError, socket.timeout): + logging.error("Unable to open URL: {}".format(url)) + raise + + +def edit_tht_env(env_file, section, settings): + assert isinstance(settings, dict) + with open(env_file) as fh: + data = yaml.safe_load(fh) + + if section not in data.keys(): + data[section] = {} + for setting, value in settings.items(): + data[section][setting] = value + with open(env_file, 'w') as fh: + yaml.safe_dump(data, fh, default_flow_style=False) + logging.debug("Data written to env file {}:\n{}".format(env_file, data)) -- cgit 1.2.3-korg