:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: : Copyright (c) 2017 Enea AB and others. : : All rights reserved. This program and the accompanying materials : are made available under the terms of the Apache License, Version 2.0 : which accompanies this distribution, and is available at : http://www.apache.org/licenses/LICENSE-2.0 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: From: Alexandru Avadanii Date: Thu, 24 Nov 2016 23:02:04 +0100 Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts Add support for caching deploy artifacts, like bootstraps and target images, which take a lot of time at each deploy to be built, considering it requires a cross-debootstrap via qemu-user-static and binfmt. For OPNFV CI, the cache will piggy back on the mechanism, and be located at: /iso_mount/opnfv_ci//deploy-cache TODO: Use dea interface adapter in target images fingerprinting. TODO: remote fingerprinting TODO: differentiate between bootstraps and targetimages, so we don't end up trying to use one cache artifact type as the other. TODO: implement sanity checks for bootstrap and target images; TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`; JIRA: ARMBAND-172 JIRA: ARMBAND-242 Signed-off-by: Alexandru Avadanii --- ...p_admin_node.sh-deploy_cache-install-hook.patch | 90 ++++++ ci/deploy.sh | 14 +- deploy/cloud/configure_settings.py | 4 + deploy/cloud/deployment.py | 12 + deploy/deploy.py | 25 +- deploy/deploy_cache.py | 314 +++++++++++++++++++++ deploy/deploy_env.py | 13 +- deploy/install_fuel_master.py | 9 +- 8 files changed, 472 insertions(+), 9 deletions(-) create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch create mode 100644 deploy/deploy_cache.py diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch new file mode 100644 index 0000000..7acb746 --- /dev/null +++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch @@ -0,0 +1,90 @@ +From: Alexandru Avadanii +Date: Mon, 28 Nov 2016 14:27:48 +0100 +Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook + +Tooling on the automatic deploy side was updated to support deploy +caching of artifacts like bootstrap (and id_rsa keypair), target +images etc. + +Add installation hook that calls `fuel-bootstrap import` instead of +`build` when a bootstrap tar is available in the agreed location, +/var/lib/opnfv/cache/bootstraps/. + +Temporary until Fuel@Openstack fixes Master key propagation to nodes' +authorized_keys, use Mcollective remote shell execute to add it +during deployment. +This might duplicate the entry in authorized_keys during re-deploys. + +JIRA: ARMBAND-172 +JIRA: ARMBAND-242 + +Signed-off-by: Alexandru Avadanii +--- + iso/bootstrap_admin_node.sh | 35 ++++++++++++++++++++++++++++++++++- + 1 file changed, 34 insertions(+), 1 deletion(-) + +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh +index 4f5ce4e..4c79552 100755 +--- a/iso/bootstrap_admin_node.sh ++++ b/iso/bootstrap_admin_node.sh +@@ -64,6 +64,8 @@ wget \ + ASTUTE_YAML='/etc/fuel/astute.yaml' + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" + CUSTOM_REPOS="/root/default_deb_repos.yaml" ++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps" ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar" + bs_build_log='/var/log/fuel-bootstrap-image-build.log' + bs_status=0 + # Backup network configs to this folder. Folder will be created only if +@@ -97,6 +99,7 @@ image becomes available, reboot nodes that failed to be discovered." + bs_done_message="Default bootstrap image building done. Now you can boot new \ + nodes over PXE, they will be discovered and become available for installing \ + OpenStack on them" ++bs_cache_message="OPNFV deploy cache: bootstrap image injected." + # Update issues messages + update_warn_message="There is an issue connecting to update repository of \ + your distributions of OpenStack. \ +@@ -509,12 +512,42 @@ set_ui_bootstrap_error () { + EOF + } + ++function inject_cached_ssh_key () { ++ # FIXME(armband): Propagate master ssh key to nodes' ++ # authorized_keys, until upstream fixes this for image build. ++ local moddir="/etc/puppet/${OPENSTACK_VERSION}/modules/osnailyfacter/modular" ++ cat >> "${moddir}/astute/generate_keys.sh" <<-EOF ++ mco rpc execute_shell_command execute \\ ++ cmd="echo $(cat /root/.ssh/id_rsa.pub) >> /root/.ssh/authorized_keys" ++ EOF ++} ++ ++function inject_cached_ubuntu_bootstrap () { ++ if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \ ++ -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \ ++ -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then ++ if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \ ++ cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \ ++ cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \ ++ sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \ ++ /etc/nailgun/settings.yaml && \ ++ fuel-bootstrap -v --debug import --activate \ ++ "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then ++ inject_cached_ssh_key ++ fuel notify --topic "done" --send "${bs_cache_message}" ++ return 0 ++ fi ++ fi ++ return 1 ++} ++ + # Actually build the bootstrap image + build_ubuntu_bootstrap () { + local ret=1 + echo ${bs_progress_message} >&2 + set_ui_bootstrap_error "${bs_progress_message}" >&2 +- if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then ++ if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \ ++ build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then + ret=0 + fuel notify --topic "done" --send "${bs_done_message}" + else diff --git a/ci/deploy.sh b/ci/deploy.sh index 081806c..4b1ae0e 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -29,7 +29,7 @@ cat << EOF xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx `basename $0`: Deploys the Fuel@OPNFV stack -usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso +usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso -s deployment-scenario [-S optional Deploy-scenario path URI] [-R optional local relen repo (containing deployment Scenarios] @@ -47,6 +47,7 @@ OPTIONS: -p Pod-name -s Deploy-scenario short-name/base-file-name -S Storage dir for VM images + -C Deploy cache dir for storing image artifacts -T Timeout, in minutes, for the deploy. -i iso url @@ -79,6 +80,7 @@ Input parameters to the build script is: or a deployment short-name as defined by scenario.yaml in the deployment scenario path. -S Storage dir for VM images, default is fuel/deploy/images +-C Deploy cache dir for bootstrap and target image artifacts, optional -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT environment variable when defined, or to the default in deploy.py otherwise -i .iso image to be deployed (needs to be provided in a URI @@ -116,6 +118,7 @@ FUEL_CREATION_ONLY='' NO_DEPLOY_ENVIRONMENT='' STORAGE_DIR='' DRY_RUN=0 +DEPLOY_CACHE_DIR='' if ! [ -z $DEPLOY_TIMEOUT ]; then DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT" else @@ -128,7 +131,7 @@ fi ############################################################################ # BEGIN of main # -while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION +while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION do case $OPTION in b) @@ -179,6 +182,9 @@ do STORAGE_DIR="-s ${OPTARG}" fi ;; + C) + DEPLOY_CACHE_DIR="-dc ${OPTARG}" + ;; T) DEPLOY_TIMEOUT="-dt ${OPTARG}" ;; @@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then ISO=${SCRIPT_PATH}/ISO/image.iso fi # Start deployment - echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT" - python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT + echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR" + python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR fi popd > /dev/null diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py index b60a60f..4e007e1 100644 --- a/deploy/cloud/configure_settings.py +++ b/deploy/cloud/configure_settings.py @@ -71,5 +71,9 @@ class ConfigureSettings(object): settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id'] settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] + # deploy-cache req: pass master id_rsa.pub as authorized key + with io.open('/root/.ssh/id_rsa.pub', 'r') as pkey: + settings['editable']['operator_user']['authkeys']['value'] = pkey.read() + with io.open(settings_yaml, 'w') as stream: yaml.dump(settings, stream, default_flow_style=False) diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py index 4329a4c..a84d46c 100644 --- a/deploy/cloud/deployment.py +++ b/deploy/cloud/deployment.py @@ -19,6 +19,8 @@ from common import ( log, ) +from deploy_cache import DeployCache + SEARCH_TEXT = '(err)' LOG_FILE = '/var/log/puppet.log' GREP_LINES_OF_LEADING_CONTEXT = 100 @@ -51,6 +53,14 @@ class Deployment(object): self.pattern = re.compile( '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d') + def deploy_cache_install_targetimages(self): + log('Using target images from deploy cache') + DeployCache.install_targetimages_for_env(self.env_id) + + def deploy_cache_extract_targetimages(self): + log('Collecting Fuel target image files for deploy cache') + DeployCache.extract_targetimages_from_env(self.env_id) + def collect_error_logs(self): for node_id, roles_blade in self.node_id_roles_dict.iteritems(): log_list = [] @@ -112,6 +122,7 @@ class Deployment(object): start = time.time() log('Starting deployment of environment %s' % self.env_id) + self.deploy_cache_install_targetimages() deploy_id = None ready = False timeout = False @@ -144,6 +155,7 @@ class Deployment(object): err('Deployment timed out, environment %s is not operational, ' 'snapshot will not be performed' % self.env_id) + self.deploy_cache_extract_targetimages() if ready: log('Environment %s successfully deployed' % self.env_id) diff --git a/deploy/deploy.py b/deploy/deploy.py index 7648baf..ee3cb7a 100755 --- a/deploy/deploy.py +++ b/deploy/deploy.py @@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter from dha import DeploymentHardwareAdapter from install_fuel_master import InstallFuelMaster from deploy_env import CloudDeploy +from deploy_cache import DeployCache from execution_environment import ExecutionEnvironment from common import ( @@ -61,7 +62,8 @@ class AutoDeploy(object): def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only, cleanup, storage_dir, pxe_bridge, iso_file, dea_file, dha_file, fuel_plugins_dir, fuel_plugins_conf_dir, - no_plugins, deploy_timeout, no_deploy_environment, deploy_log): + no_plugins, deploy_cache_dir, deploy_timeout, + no_deploy_environment, deploy_log): self.no_fuel = no_fuel self.fuel_only = fuel_only self.no_health_check = no_health_check @@ -75,6 +77,7 @@ class AutoDeploy(object): self.fuel_plugins_dir = fuel_plugins_dir self.fuel_plugins_conf_dir = fuel_plugins_conf_dir self.no_plugins = no_plugins + self.deploy_cache_dir = deploy_cache_dir self.deploy_timeout = deploy_timeout self.no_deploy_environment = no_deploy_environment self.deploy_log = deploy_log @@ -116,7 +119,7 @@ class AutoDeploy(object): self.fuel_username, self.fuel_password, self.dea_file, self.fuel_plugins_conf_dir, WORK_DIR, self.no_health_check, - self.deploy_timeout, + self.deploy_cache_dir, self.deploy_timeout, self.no_deploy_environment, self.deploy_log) with old_dep.ssh: old_dep.check_previous_installation() @@ -128,6 +131,7 @@ class AutoDeploy(object): self.fuel_conf['ip'], self.fuel_username, self.fuel_password, self.fuel_node_id, self.iso_file, WORK_DIR, + self.deploy_cache_dir, self.fuel_plugins_dir, self.no_plugins) fuel.install() @@ -136,6 +140,7 @@ class AutoDeploy(object): tmp_new_dir = '%s/newiso' % self.tmp_dir try: self.copy(tmp_orig_dir, tmp_new_dir) + self.deploy_cache_fingerprints(tmp_new_dir) self.patch(tmp_new_dir, new_iso) except Exception as e: exec_cmd('fusermount -u %s' % tmp_orig_dir, False) @@ -156,6 +161,12 @@ class AutoDeploy(object): delete(tmp_orig_dir) exec_cmd('chmod -R 755 %s' % tmp_new_dir) + def deploy_cache_fingerprints(self, tmp_new_dir): + if self.deploy_cache_dir: + log('Deploy cache: Collecting fingerprints...') + deploy_cache = DeployCache(self.deploy_cache_dir) + deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file) + def patch(self, tmp_new_dir, new_iso): log('Patching...') patch_dir = '%s/%s' % (CWD, PATCH_DIR) @@ -218,7 +229,8 @@ class AutoDeploy(object): dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'], self.fuel_username, self.fuel_password, self.dea_file, self.fuel_plugins_conf_dir, - WORK_DIR, self.no_health_check, self.deploy_timeout, + WORK_DIR, self.no_health_check, + self.deploy_cache_dir, self.deploy_timeout, self.no_deploy_environment, self.deploy_log) return dep.deploy() @@ -343,6 +355,8 @@ def parse_arguments(): help='Fuel Plugins Configuration directory') parser.add_argument('-np', dest='no_plugins', action='store_true', default=False, help='Do not install Fuel Plugins') + parser.add_argument('-dc', dest='deploy_cache_dir', action='store', + help='Deploy Cache Directory') parser.add_argument('-dt', dest='deploy_timeout', action='store', default=240, help='Deployment timeout (in minutes) ' '[default: 240]') @@ -376,6 +390,10 @@ def parse_arguments(): for bridge in args.pxe_bridge: check_bridge(bridge, args.dha_file) + if args.deploy_cache_dir: + log('Using deploy cache directory: %s' % args.deploy_cache_dir) + create_dir_if_not_exists(args.deploy_cache_dir) + kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only, 'no_health_check': args.no_health_check, @@ -386,6 +404,7 @@ def parse_arguments(): 'fuel_plugins_dir': args.fuel_plugins_dir, 'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir, 'no_plugins': args.no_plugins, + 'deploy_cache_dir': args.deploy_cache_dir, 'deploy_timeout': args.deploy_timeout, 'no_deploy_environment': args.no_deploy_environment, 'deploy_log': args.deploy_log} diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py new file mode 100644 index 0000000..30bfe30 --- /dev/null +++ b/deploy/deploy_cache.py @@ -0,0 +1,314 @@ +############################################################################### +# Copyright (c) 2016 Enea AB and others. +# Alexandru.Avadanii@enea.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################### + +import glob +import hashlib +import io +import json +import os +import shutil +import yaml + +from common import ( + exec_cmd, + log, +) + +############################################################################### +# Deploy Cache Flow Overview +############################################################################### +# 1. do_fingerprints +# Can be called as soon as a Fuel Master ISO chroot is available. +# This will gather all required information for uniquely identifying the +# objects in cache (bootstraps, targetimages). +# 2. inject_cache +# Can be called as soon as we have a steady SSH connection to the Fuel +# Master node. It will inject cached artifacts over SSH, for later install. +# 3. (external, async) install cached bootstrap instead of building a new one +# /sbin/bootstrap_admin_node.sh will check for cached bootstrap images +# (with id_rsa, id_rsa.pub attached) and will install those via +# $ fuel-bootstrap import opfnv-bootstraps-cache.tar +# 4. install_targetimages_for_env +# Should be called before cloud deploy is started, to install env-generic +# 'env_X_...' cached images for the current environment ID. +# Static method, to be used on the remote Fuel Master node; does not require +# access to the deploy cache, it only moves around some local files. +# 5. extract_targetimages_from_env +# Should be called at env deploy finish, to prepare artifacts for caching. +# Static method, same observations as above apply. +# 6. collect_artifacts +# Call last, to collect all artifacts. +############################################################################### + +############################################################################### +# Deploy cache artifacts: +# - id_rsa +# - bootstrap image (Ubuntu) +# - environment target image (Ubuntu) +############################################################################### +# Cache fingerprint covers: +# - bootstrap: +# - local mirror contents +# - FIXME(armband): [disabled] package list (old fuel_bootstrap_cli.yaml) +# - target image: +# - local mirror contents +# - package list (determined from DEA) +############################################################################### +# WARN: Cache fingerprint does NOT yet cover: +# - image_data (always assume the default /boot, /); +# - output_dir (always assume the default /var/www/nailgun/targetimages; +# - codename (always assume the default, currently 'trusty'); +# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty +# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub +# - Auxiliary repo .../mitaka-9.0/ubuntu/auxiliary +# If the above change without triggering a cache miss, try clearing the cache. +############################################################################### +# WARN: Bootstrap caching implies RSA keypair to be reused! +############################################################################### + +# Local mirrros will be used on Fuel Master for both bootstrap and target image +# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`: +# - MOS .../mitaka-9.0/ubuntu/x86_64 +# - Ubuntu .../mirrors/ubuntu/ +# All these reside on Fuel Master at local path: +NAILGUN_PATH = '/var/www/nailgun/' + +# Artifact names (corresponding to nailgun subdirs) +MIRRORS = 'mirrors' +BOOTSTRAPS = 'bootstraps' +TARGETIMAGES = 'targetimages' + +# Info for collecting RSA keypair +RSA_KEYPAIR_PATH = '/root/.ssh' +RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub'] + +# Relative path for collecting the active bootstrap image(s) after env deploy +NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS + +# Relative path for collecting target image(s) for deployed enviroment +NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES + +# FIXME(armband): re-include package list (old fuel_bootstrap_cli.yaml) +# ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml' + +# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected +REMOTE_CACHE_PATH = '/var/cache/opnfv' + +# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh +BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar' + +# Env-ID indep prefix +ENVX = 'env_X_' + +class DeployCache(object): + """OPNFV Deploy Cache - managed storage for cacheable artifacts""" + + def __init__(self, cache_dir, + fingerprints_yaml='deploy_cache_fingerprints.yaml'): + self.cache_dir = cache_dir + self.fingerprints_yaml = fingerprints_yaml + self.fingerprints = {BOOTSTRAPS: None, + MIRRORS: None, + TARGETIMAGES: None} + + def __load_fingerprints(self): + """Load deploy cache yaml config holding fingerprints""" + if os.path.isfile(self.fingerprints_yaml): + cache_fingerprints = open(self.fingerprints_yaml).read() + self.fingerprints = yaml.load(cache_fingerprints) + + def __save_fingerprints(self): + """Update deploy cache yaml config holding fingerprints""" + with open(self.fingerprints_yaml, 'w') as outfile: + outfile.write(yaml.safe_dump(self.fingerprints, + default_flow_style=False)) + + def __fingerprint_mirrors(self, chroot_path): + """Collect repo mirror fingerprints""" + deb_packages = list() + # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins) + for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']: + for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)): + for fdeb in files: + if fdeb.endswith(".deb"): + deb_packages.append(fdeb) + sorted_debs = json.dumps(deb_packages, sort_keys=True) + self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest() + + def __fingerprint_bootstrap(self, chroot_path): + """Collect bootstrap image metadata fingerprints""" + # FIXME(armband): include 'extra_dirs' contents + sorted_data = '' + # FIXME(armband): re-include package list (old fuel_bootstrap_cli.yaml) + # cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:]) + # bootstrap_cli_yaml = open(cli_yaml_path).read() + # bootstrap_data = yaml.load(bootstrap_cli_yaml) + # sorted_data = json.dumps(bootstrap_data, sort_keys=True) + self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest() + + def __fingerprint_target(self, dea_file): + """Collect target image metadata fingerprints""" + # FIXME(armband): include 'image_data', 'codename', 'output' + with io.open(dea_file) as stream: + dea = yaml.load(stream) + editable = dea['settings']['editable'] + target_data = {'packages': editable['provision']['packages'], + 'repos': editable['repo_setup']['repos']} + s_data = json.dumps(target_data, sort_keys=True) + self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest() + + def do_fingerprints(self, chroot_path, dea_file): + """Collect SHA1 fingerprints based on chroot contents, DEA settings""" + try: + self.__load_fingerprints() + self.__fingerprint_mirrors(chroot_path) + self.__fingerprint_bootstrap(chroot_path) + self.__fingerprint_target(dea_file) + self.__save_fingerprints() + except Exception as ex: + log('Failed to get cache fingerprint: %s' % str(ex)) + + def __lookup_cache(self, sha): + """Search for object in cache based on SHA fingerprint""" + cache_sha_dir = os.path.join(self.cache_dir, sha) + if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir): + return None + return cache_sha_dir + + def __inject_cache_dir(self, ssh, sha, artifact): + """Stage cached object (dir) in Fuel Master OPNFV local cache""" + local_path = self.__lookup_cache(sha) + if local_path: + remote_path = os.path.join(REMOTE_CACHE_PATH, artifact) + with ssh: + ssh.exec_cmd('mkdir -p %s' % remote_path) + for cachedfile in glob.glob('%s/*' % local_path): + ssh.scp_put(cachedfile, remote_path) + return local_path + + def __mix_fingerprints(self, f1, f2): + """Compute composite fingerprint""" + if self.fingerprints[f1] is None or self.fingerprints[f2] is None: + return None + return hashlib.sha1('%s%s' % + (self.fingerprints[f1], self.fingerprints[f2])).hexdigest() + + def inject_cache(self, ssh): + """Lookup artifacts in cache and inject them over SSH/SCP into Fuel""" + try: + self.__load_fingerprints() + for artifact in [BOOTSTRAPS, TARGETIMAGES]: + sha = self.__mix_fingerprints(MIRRORS, artifact) + if sha is None: + log('Missing fingerprint for: %s' % artifact) + continue + if not self.__inject_cache_dir(ssh, sha, artifact): + log('SHA1 not in cache: %s (%s)' % (str(sha), artifact)) + else: + log('SHA1 injected: %s (%s)' % (str(sha), artifact)) + except Exception as ex: + log('Failed to inject cached artifacts into Fuel: %s' % str(ex)) + + def __extract_bootstraps(self, ssh, cache_sha_dir): + """Collect bootstrap artifacts from Fuel over SSH/SCP""" + remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE) + local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE) + with ssh: + for k in RSA_KEYPAIR_FILES: + ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k), + local=os.path.join(cache_sha_dir, k)) + ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' % + (REMOTE_CACHE_PATH, + os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR), + remote_tar)) + ssh.scp_get(remote_tar, local=local_tar) + ssh.exec_cmd('rm -f %s' % remote_tar) + + def __extract_targetimages(self, ssh, cache_sha_dir): + """Collect target image artifacts from Fuel over SSH/SCP""" + cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) + with ssh: + ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir) + + def collect_artifacts(self, ssh): + """Collect artifacts from Fuel over SSH/SCP and add them to cache""" + try: + self.__load_fingerprints() + for artifact, func in { + BOOTSTRAPS: self.__extract_bootstraps, + TARGETIMAGES: self.__extract_targetimages + }.iteritems(): + sha = self.__mix_fingerprints(MIRRORS, artifact) + if sha is None: + log('WARN: Skip caching, NO fingerprint: %s' % artifact) + continue + local_path = self.__lookup_cache(sha) + if local_path: + log('SHA1 already in cache: %s (%s)' % (str(sha), artifact)) + else: + log('New cache SHA1: %s (%s)' % (str(sha), artifact)) + cache_sha_dir = os.path.join(self.cache_dir, sha) + exec_cmd('mkdir -p %s' % cache_sha_dir) + func(ssh, cache_sha_dir) + except Exception as ex: + log('Failed to extract artifacts from Fuel: %s' % str(ex)) + + @staticmethod + def extract_targetimages_from_env(env_id): + """Prepare targetimages from env ID for storage in deploy cache + + NOTE: This method should be executed locally ON the Fuel Master node. + WARN: This method overwrites targetimages cache on Fuel Master node. + """ + env_n = 'env_%s_' % str(env_id) + cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) + ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR) + try: + exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path)) + for root, _, files in os.walk(ti_path): + for tif in files: + if tif.startswith(env_n): + src = os.path.join(root, tif) + dest = os.path.join(cti_path, tif.replace(env_n, ENVX)) + if tif.endswith('.yaml'): + shutil.copy(src, dest) + exec_cmd('sed -i "s|%s|%s|g" %s' % + (env_n, ENVX, dest)) + else: + os.link(src, dest) + except Exception as ex: + log('Failed to extract targetimages artifacts from env %s: %s' % + (str(env_id), str(ex))) + + @staticmethod + def install_targetimages_for_env(env_id): + """Install targetimages artifacts for a specific env ID + + NOTE: This method should be executed locally ON the Fuel Master node. + """ + env_n = 'env_%s_' % str(env_id) + cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) + ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR) + if not os.path.isdir(cti_path): + log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path)) + else: + try: + for root, _, files in os.walk(cti_path): + for tif in files: + src = os.path.join(root, tif) + dest = os.path.join(ti_path, tif.replace(ENVX, env_n)) + if tif.endswith('.yaml'): + shutil.copy(src, dest) + exec_cmd('sed -i "s|%s|%s|g" %s' % + (ENVX, env_n, dest)) + else: + os.link(src, dest) + except Exception as ex: + log('Failed to install targetimages for env %s: %s' % + (str(env_id), str(ex))) diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py index d374cce..445070a 100644 --- a/deploy/deploy_env.py +++ b/deploy/deploy_env.py @@ -15,6 +15,7 @@ import glob import time import shutil +from deploy_cache import DeployCache from ssh_client import SSHClient from common import ( @@ -35,7 +36,8 @@ class CloudDeploy(object): def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password, dea_file, fuel_plugins_conf_dir, work_dir, no_health_check, - deploy_timeout, no_deploy_environment, deploy_log): + deploy_cache_dir, deploy_timeout, + no_deploy_environment, deploy_log): self.dea = dea self.dha = dha self.fuel_ip = fuel_ip @@ -49,6 +51,8 @@ class CloudDeploy(object): self.fuel_plugins_conf_dir = fuel_plugins_conf_dir self.work_dir = work_dir self.no_health_check = no_health_check + self.deploy_cache = ( DeployCache(deploy_cache_dir) + if deploy_cache_dir else None ) self.deploy_timeout = deploy_timeout self.no_deploy_environment = no_deploy_environment self.deploy_log = deploy_log @@ -82,9 +86,14 @@ class CloudDeploy(object): self.work_dir, os.path.basename(self.dea_file))) s.scp_put('%s/common.py' % self.file_dir, self.work_dir) s.scp_put('%s/dea.py' % self.file_dir, self.work_dir) + s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir) for f in glob.glob('%s/cloud/*' % self.file_dir): s.scp_put(f, self.work_dir) + def deploy_cache_collect_artifacts(self): + if self.deploy_cache: + self.deploy_cache.collect_artifacts(self.ssh) + def power_off_nodes(self): for node_id in self.node_ids: self.dha.node_power_off(node_id) @@ -281,4 +290,6 @@ class CloudDeploy(object): self.get_put_deploy_log() + self.deploy_cache_collect_artifacts() + return rc diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py index b731c6b..83d31fb 100644 --- a/deploy/install_fuel_master.py +++ b/deploy/install_fuel_master.py @@ -10,6 +10,7 @@ import time import os import glob +from deploy_cache import DeployCache from ssh_client import SSHClient from dha_adapters.libvirt_adapter import LibvirtAdapter @@ -32,7 +33,7 @@ class InstallFuelMaster(object): def __init__(self, dea_file, dha_file, fuel_ip, fuel_username, fuel_password, fuel_node_id, iso_file, work_dir, - fuel_plugins_dir, no_plugins): + deploy_cache_dir, fuel_plugins_dir, no_plugins): self.dea_file = dea_file self.dha = LibvirtAdapter(dha_file) self.fuel_ip = fuel_ip @@ -42,6 +43,8 @@ class InstallFuelMaster(object): self.iso_file = iso_file self.iso_dir = os.path.dirname(self.iso_file) self.work_dir = work_dir + self.deploy_cache = ( DeployCache(deploy_cache_dir) + if deploy_cache_dir else None ) self.fuel_plugins_dir = fuel_plugins_dir self.no_plugins = no_plugins self.file_dir = os.path.dirname(os.path.realpath(__file__)) @@ -83,6 +86,10 @@ class InstallFuelMaster(object): log('Wait until Fuel menu is up') fuel_menu_pid = self.wait_until_fuel_menu_up() + if self.deploy_cache: + log('Deploy cache: Injecting bootstraps and targetimages') + self.deploy_cache.inject_cache(self.ssh) + log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings') self.inject_own_astute_and_bootstrap_yaml()