diff options
Diffstat (limited to 'apex/common')
-rw-r--r-- | apex/common/constants.py | 43 | ||||
-rw-r--r-- | apex/common/exceptions.py | 16 | ||||
-rw-r--r-- | apex/common/utils.py | 180 |
3 files changed, 207 insertions, 32 deletions
diff --git a/apex/common/constants.py b/apex/common/constants.py index a2b9a634..59988f74 100644 --- a/apex/common/constants.py +++ b/apex/common/constants.py @@ -16,7 +16,7 @@ STORAGE_NETWORK = 'storage' API_NETWORK = 'api' CONTROLLER = 'controller' COMPUTE = 'compute' - +ANSIBLE_PATH = 'ansible/playbooks' OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, TENANT_NETWORK, EXTERNAL_NETWORK, STORAGE_NETWORK, API_NETWORK] DNS_SERVERS = ["8.8.8.8", "8.8.4.4"] @@ -39,14 +39,43 @@ VIRT_PW = '--root-password' THT_DIR = '/usr/share/openstack-tripleo-heat-templates' THT_ENV_DIR = os.path.join(THT_DIR, 'environments') +THT_DOCKER_ENV_DIR = os.path.join(THT_ENV_DIR, 'services') -DEFAULT_OS_VERSION = 'pike' -DEFAULT_ODL_VERSION = 'nitrogen' -VALID_ODL_VERSIONS = ['carbon', 'nitrogen', 'oxygen', 'master'] +DEFAULT_OS_VERSION = 'master' +DEFAULT_ODL_VERSION = 'oxygen' +VALID_ODL_VERSIONS = ['carbon', 'nitrogen', 'oxygen', 'fluorine', + 'neon', 'master'] PUPPET_ODL_URL = 'https://git.opendaylight.org/gerrit/integration/packaging' \ '/puppet-opendaylight' DEBUG_OVERCLOUD_PW = 'opnfvapex' NET_ENV_FILE = 'network-environment.yaml' -DEPLOY_TIMEOUT = 90 -UPSTREAM_RDO = 'https://images.rdoproject.org/pike/delorean/current-tripleo/' -OPENSTACK_GERRIT = 'https://review.openstack.org' +DEPLOY_TIMEOUT = 120 +RDO_TAG = 'current-tripleo' +UPSTREAM_RDO = "https://images.rdoproject.org/master/rdo_trunk/{}/".format( + RDO_TAG) +OPENSTACK_GERRIT = 'https://review.opendev.org' + +DOCKER_TAG = RDO_TAG +# Maps regular service files to docker versions +# None value means mapping is same as key +VALID_DOCKER_SERVICES = { + 'neutron-opendaylight.yaml': None, + 'neutron-opendaylight-dpdk.yaml': None, + 'neutron-opendaylight-sriov.yaml': None, + 'neutron-bgpvpn-opendaylight.yaml': None, + 'neutron-sfc-opendaylight.yaml': None, + 'neutron-ml2-ovn.yaml': 'neutron-ovn-ha.yaml' +} +DOCKERHUB_OOO = 'https://registry.hub.docker.com/v2/repositories' \ + '/tripleomaster/' +DOCKERHUB_AARCH64 = 'https://registry.hub.docker.com/v2/repositories' \ + '/armbandapex/' +KUBESPRAY_URL = 'https://github.com/kubernetes-incubator/kubespray.git' +OPNFV_ARTIFACTS = 'http://storage.googleapis.com/artifacts.opnfv.org' +CUSTOM_OVS = '{}/apex/random/openvswitch-2.9.0-9.el7fdn.x86_64.' \ + 'rpm'.format(OPNFV_ARTIFACTS) + +OVS_URL = "http://openvswitch.org/releases/openvswitch-2.9.2.tar.gz" +QUAGGA_URL = "{}/sdnvpn/quagga/quagga-4.tar.gz".format(OPNFV_ARTIFACTS) +CALIPSO_INSTALLER_URL = "https://raw.githubusercontent.com/opnfv/calipso" \ + "/master/app/install/calipso-installer.py" diff --git a/apex/common/exceptions.py b/apex/common/exceptions.py index 54d99834..6d8383b8 100644 --- a/apex/common/exceptions.py +++ b/apex/common/exceptions.py @@ -18,3 +18,19 @@ class JumpHostNetworkException(Exception): class ApexCleanException(Exception): pass + + +class ApexBuildException(Exception): + pass + + +class SnapshotDeployException(Exception): + pass + + +class OvercloudNodeException(Exception): + pass + + +class FetchException(Exception): + pass diff --git a/apex/common/utils.py b/apex/common/utils.py index 13250a45..72a66d10 100644 --- a/apex/common/utils.py +++ b/apex/common/utils.py @@ -8,10 +8,12 @@ ############################################################################## import datetime +import distro import json import logging import os import pprint +import socket import subprocess import tarfile import time @@ -20,6 +22,8 @@ import urllib.request import urllib.parse import yaml +from apex.common import exceptions as exc + def str2bool(var): if isinstance(var, bool): @@ -71,12 +75,17 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root', Executes ansible playbook and checks for errors :param ansible_vars: dictionary of variables to inject into ansible run :param playbook: playbook to execute + :param host: inventory file or string of target hosts + :param user: remote user to run ansible tasks :param tmp_dir: temp directory to store ansible command :param dry_run: Do not actually apply changes :return: None """ logging.info("Executing ansible playbook: {}".format(playbook)) - inv_host = "{},".format(host) + if not os.path.isfile(host): + inv_host = "{},".format(host) + else: + inv_host = host if host == 'localhost': conn_type = 'local' else: @@ -137,45 +146,66 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root', raise Exception(e) -def fetch_upstream_and_unpack(dest, url, targets): +def get_url_modified_date(url): + """ + Returns the last modified date for an Tripleo image artifact + :param url: URL to examine + :return: datetime object of when artifact was last modified + """ + try: + u = urllib.request.urlopen(url) + except urllib.error.URLError as e: + logging.error("Failed to fetch target url. Error: {}".format( + e.reason)) + raise + + metadata = u.info() + headers = metadata.items() + for header in headers: + if isinstance(header, tuple) and len(header) == 2: + if header[0] == 'Last-Modified': + return datetime.datetime.strptime(header[1], + "%a, %d %b %Y %X GMT") + + +def fetch_upstream_and_unpack(dest, url, targets, fetch=True): """ Fetches targets from a url destination and downloads them if they are newer. Also unpacks tar files in dest dir. :param dest: Directory to download and unpack files to :param url: URL where target files are located :param targets: List of target files to download + :param fetch: Whether or not to fetch latest from internet (boolean) :return: None """ os.makedirs(dest, exist_ok=True) assert isinstance(targets, list) for target in targets: - download_target = True target_url = urllib.parse.urljoin(url, target) target_dest = os.path.join(dest, target) - logging.debug("Fetching and comparing upstream target: \n{}".format( - target_url)) - try: - u = urllib.request.urlopen(target_url) - except urllib.error.URLError as e: - logging.error("Failed to fetch target url. Error: {}".format( - e.reason)) - raise - if os.path.isfile(target_dest): + target_exists = os.path.isfile(target_dest) + if fetch: + download_target = True + elif not target_exists: + logging.warning("no-fetch requested but target: {} is not " + "cached, will download".format(target_dest)) + download_target = True + else: + logging.info("no-fetch requested and previous cache exists for " + "target: {}. Will skip download".format(target_dest)) + download_target = False + + if download_target: + logging.debug("Fetching and comparing upstream" + " target: \n{}".format(target_url)) + # Check if previous file and fetch we need to compare files to + # determine if download is necessary + if target_exists and download_target: logging.debug("Previous file found: {}".format(target_dest)) - metadata = u.info() - headers = metadata.items() - target_url_date = None - for header in headers: - if isinstance(header, tuple) and len(header) == 2: - if header[0] == 'Last-Modified': - target_url_date = header[1] - break + target_url_date = get_url_modified_date(target_url) if target_url_date is not None: target_dest_mtime = os.path.getmtime(target_dest) - target_url_mtime = time.mktime( - datetime.datetime.strptime(target_url_date, - "%a, %d %b %Y %X " - "GMT").timetuple()) + target_url_mtime = time.mktime(target_url_date.timetuple()) if target_url_mtime > target_dest_mtime: logging.debug('URL target is newer than disk...will ' 'download') @@ -184,11 +214,111 @@ def fetch_upstream_and_unpack(dest, url, targets): download_target = False else: logging.debug('Unable to find last modified url date') + if download_target: urllib.request.urlretrieve(target_url, filename=target_dest) logging.info("Target downloaded: {}".format(target)) - if target.endswith('.tar'): + if target.endswith(('.tar', 'tar.gz', 'tgz')): logging.info('Unpacking tar file') tar = tarfile.open(target_dest) tar.extractall(path=dest) tar.close() + + +def install_ansible(): + # we only install for CentOS/Fedora for now + dist = distro.id() + if 'centos' in dist: + pkg_mgr = 'yum' + elif 'fedora' in dist: + pkg_mgr = 'dnf' + else: + return + + # yum python module only exists for 2.x, so use subprocess + try: + subprocess.check_call([pkg_mgr, '-y', 'install', 'ansible']) + except subprocess.CalledProcessError: + logging.warning('Unable to install Ansible') + + +def internet_connectivity(): + try: + urllib.request.urlopen('http://opnfv.org', timeout=3) + return True + except (urllib.request.URLError, socket.timeout): + logging.debug('No internet connectivity detected') + return False + + +def open_webpage(url, timeout=5): + try: + response = urllib.request.urlopen(url, timeout=timeout) + return response.read() + except (urllib.request.URLError, socket.timeout) as e: + logging.error("Unable to open URL: {}".format(url)) + raise exc.FetchException('Unable to open URL') from e + + +def edit_tht_env(env_file, section, settings): + assert isinstance(settings, dict) + with open(env_file) as fh: + data = yaml.safe_load(fh) + + if section not in data.keys(): + data[section] = {} + for setting, value in settings.items(): + data[section][setting] = value + with open(env_file, 'w') as fh: + yaml.safe_dump(data, fh, default_flow_style=False) + logging.debug("Data written to env file {}:\n{}".format(env_file, data)) + + +def unique(tmp_list): + assert isinstance(tmp_list, list) + uniq_list = [] + for x in tmp_list: + if x not in uniq_list: + uniq_list.append(x) + return uniq_list + + +def bash_settings_to_dict(data): + """ + Parses bash settings x=y and returns dict of key, values + :param data: bash settings data in x=y format + :return: dict of keys and values + """ + return dict(item.split('=') for item in data.splitlines()) + + +def fetch_properties(url): + """ + Downloads OPNFV properties and returns a dictionary of the key, values + :param url: URL of properties file + :return: dict of k,v for each properties + """ + if bool(urllib.parse.urlparse(url).scheme): + logging.debug('Fetching properties from internet: {}'.format(url)) + return bash_settings_to_dict(open_webpage(url).decode('utf-8')) + elif os.path.isfile(url): + logging.debug('Fetching properties from file: {}'.format(url)) + with open(url, 'r') as fh: + data = fh.read() + return bash_settings_to_dict(data) + else: + logging.warning('Unable to fetch properties for: {}'.format(url)) + raise exc.FetchException('Unable determine properties location: ' + '{}'.format(url)) + + +def find_container_client(os_version): + """ + Determines whether to use docker or podman client + :param os_version: openstack version + :return: client name as string + """ + if os_version == 'rocky' or os_version == 'queens': + return 'docker' + else: + return 'podman' |