diff options
Diffstat (limited to 'patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch')
| -rw-r--r-- | patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch | 786 |
1 files changed, 0 insertions, 786 deletions
diff --git a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch b/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch deleted file mode 100644 index 9997dedf..00000000 --- a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch +++ /dev/null @@ -1,786 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2017 Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 24 Nov 2016 23:02:04 +0100 -Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts - -Add support for caching deploy artifacts, like bootstraps and -target images, which take a lot of time at each deploy to be built, -considering it requires a cross-debootstrap via qemu-user-static and -binfmt. - -For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism, -and be located at: -/iso_mount/opnfv_ci/<branch>/deploy-cache - -TODO: Use dea interface adapter in target images fingerprinting. -TODO: remote fingerprinting -TODO: differentiate between bootstraps and targetimages, so we don't -end up trying to use one cache artifact type as the other. -TODO: implement sanity checks for bootstrap and target images; -TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`; - -JIRA: ARMBAND-172 -JIRA: ARMBAND-242 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - ...p_admin_node.sh-deploy_cache-install-hook.patch | 90 ++++++ - ci/deploy.sh | 14 +- - deploy/cloud/configure_settings.py | 4 + - deploy/cloud/deployment.py | 12 + - deploy/deploy.py | 25 +- - deploy/deploy_cache.py | 314 +++++++++++++++++++++ - deploy/deploy_env.py | 13 +- - deploy/install_fuel_master.py | 9 +- - 8 files changed, 472 insertions(+), 9 deletions(-) - create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch - create mode 100644 deploy/deploy_cache.py - -diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch -new file mode 100644 -index 0000000..7acb746 ---- /dev/null -+++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch -@@ -0,0 +1,90 @@ -+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -+Date: Mon, 28 Nov 2016 14:27:48 +0100 -+Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook -+ -+Tooling on the automatic deploy side was updated to support deploy -+caching of artifacts like bootstrap (and id_rsa keypair), target -+images etc. -+ -+Add installation hook that calls `fuel-bootstrap import` instead of -+`build` when a bootstrap tar is available in the agreed location, -+/var/lib/opnfv/cache/bootstraps/. -+ -+Temporary until Fuel@Openstack fixes Master key propagation to nodes' -+authorized_keys, use Mcollective remote shell execute to add it -+during deployment. -+This might duplicate the entry in authorized_keys during re-deploys. -+ -+JIRA: ARMBAND-172 -+JIRA: ARMBAND-242 -+ -+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -+--- -+ iso/bootstrap_admin_node.sh | 35 ++++++++++++++++++++++++++++++++++- -+ 1 file changed, 34 insertions(+), 1 deletion(-) -+ -+diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh -+index 4f5ce4e..4c79552 100755 -+--- a/iso/bootstrap_admin_node.sh -++++ b/iso/bootstrap_admin_node.sh -+@@ -64,6 +64,8 @@ wget \ -+ ASTUTE_YAML='/etc/fuel/astute.yaml' -+ BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" -+ CUSTOM_REPOS="/root/default_deb_repos.yaml" -++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps" -++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar" -+ bs_build_log='/var/log/fuel-bootstrap-image-build.log' -+ bs_status=0 -+ # Backup network configs to this folder. Folder will be created only if -+@@ -97,6 +99,7 @@ image becomes available, reboot nodes that failed to be discovered." -+ bs_done_message="Default bootstrap image building done. Now you can boot new \ -+ nodes over PXE, they will be discovered and become available for installing \ -+ OpenStack on them" -++bs_cache_message="OPNFV deploy cache: bootstrap image injected." -+ # Update issues messages -+ update_warn_message="There is an issue connecting to update repository of \ -+ your distributions of OpenStack. \ -+@@ -509,12 +512,42 @@ set_ui_bootstrap_error () { -+ EOF -+ } -+ -++function inject_cached_ssh_key () { -++ # FIXME(armband): Propagate master ssh key to nodes' -++ # authorized_keys, until upstream fixes this for image build. -++ local moddir="/etc/puppet/${OPENSTACK_VERSION}/modules/osnailyfacter/modular" -++ cat >> "${moddir}/astute/generate_keys.sh" <<-EOF -++ mco rpc execute_shell_command execute \\ -++ cmd="echo $(cat /root/.ssh/id_rsa.pub) >> /root/.ssh/authorized_keys" -++ EOF -++} -++ -++function inject_cached_ubuntu_bootstrap () { -++ if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \ -++ -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \ -++ -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then -++ if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \ -++ cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \ -++ cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \ -++ sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \ -++ /etc/nailgun/settings.yaml && \ -++ fuel-bootstrap -v --debug import --activate \ -++ "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then -++ inject_cached_ssh_key -++ fuel notify --topic "done" --send "${bs_cache_message}" -++ return 0 -++ fi -++ fi -++ return 1 -++} -++ -+ # Actually build the bootstrap image -+ build_ubuntu_bootstrap () { -+ local ret=1 -+ echo ${bs_progress_message} >&2 -+ set_ui_bootstrap_error "${bs_progress_message}" >&2 -+- if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then -++ if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \ -++ build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then -+ ret=0 -+ fuel notify --topic "done" --send "${bs_done_message}" -+ else -diff --git a/ci/deploy.sh b/ci/deploy.sh -index 081806c..4b1ae0e 100755 ---- a/ci/deploy.sh -+++ b/ci/deploy.sh -@@ -29,7 +29,7 @@ cat << EOF - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - `basename $0`: Deploys the Fuel@OPNFV stack - --usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso -+usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso - -s deployment-scenario [-S optional Deploy-scenario path URI] - [-R optional local relen repo (containing deployment Scenarios] - -@@ -47,6 +47,7 @@ OPTIONS: - -p Pod-name - -s Deploy-scenario short-name/base-file-name - -S Storage dir for VM images -+ -C Deploy cache dir for storing image artifacts - -T Timeout, in minutes, for the deploy. - -i iso url - -@@ -79,6 +80,7 @@ Input parameters to the build script is: - or a deployment short-name as defined by scenario.yaml in the deployment - scenario path. - -S Storage dir for VM images, default is fuel/deploy/images -+-C Deploy cache dir for bootstrap and target image artifacts, optional - -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT - environment variable when defined, or to the default in deploy.py otherwise - -i .iso image to be deployed (needs to be provided in a URI -@@ -116,6 +118,7 @@ FUEL_CREATION_ONLY='' - NO_DEPLOY_ENVIRONMENT='' - STORAGE_DIR='' - DRY_RUN=0 -+DEPLOY_CACHE_DIR='' - if ! [ -z $DEPLOY_TIMEOUT ]; then - DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT" - else -@@ -128,7 +131,7 @@ fi - ############################################################################ - # BEGIN of main - # --while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION -+while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION - do - case $OPTION in - b) -@@ -179,6 +182,9 @@ do - STORAGE_DIR="-s ${OPTARG}" - fi - ;; -+ C) -+ DEPLOY_CACHE_DIR="-dc ${OPTARG}" -+ ;; - T) - DEPLOY_TIMEOUT="-dt ${OPTARG}" - ;; -@@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then - ISO=${SCRIPT_PATH}/ISO/image.iso - fi - # Start deployment -- echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT" -- python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT -+ echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR" -+ python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR - fi - popd > /dev/null - -diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py -index b60a60f..4e007e1 100644 ---- a/deploy/cloud/configure_settings.py -+++ b/deploy/cloud/configure_settings.py -@@ -71,5 +71,9 @@ class ConfigureSettings(object): - settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id'] - settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] - -+ # deploy-cache req: pass master id_rsa.pub as authorized key -+ with io.open('/root/.ssh/id_rsa.pub', 'r') as pkey: -+ settings['editable']['operator_user']['authkeys']['value'] = pkey.read() -+ - with io.open(settings_yaml, 'w') as stream: - yaml.dump(settings, stream, default_flow_style=False) -diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py -index 4329a4c..a84d46c 100644 ---- a/deploy/cloud/deployment.py -+++ b/deploy/cloud/deployment.py -@@ -19,6 +19,8 @@ from common import ( - log, - ) - -+from deploy_cache import DeployCache -+ - SEARCH_TEXT = '(err)' - LOG_FILE = '/var/log/puppet.log' - GREP_LINES_OF_LEADING_CONTEXT = 100 -@@ -51,6 +53,14 @@ class Deployment(object): - self.pattern = re.compile( - '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d') - -+ def deploy_cache_install_targetimages(self): -+ log('Using target images from deploy cache') -+ DeployCache.install_targetimages_for_env(self.env_id) -+ -+ def deploy_cache_extract_targetimages(self): -+ log('Collecting Fuel target image files for deploy cache') -+ DeployCache.extract_targetimages_from_env(self.env_id) -+ - def collect_error_logs(self): - for node_id, roles_blade in self.node_id_roles_dict.iteritems(): - log_list = [] -@@ -112,6 +122,7 @@ class Deployment(object): - start = time.time() - - log('Starting deployment of environment %s' % self.env_id) -+ self.deploy_cache_install_targetimages() - deploy_id = None - ready = False - timeout = False -@@ -144,6 +155,7 @@ class Deployment(object): - err('Deployment timed out, environment %s is not operational, ' - 'snapshot will not be performed' - % self.env_id) -+ self.deploy_cache_extract_targetimages() - if ready: - log('Environment %s successfully deployed' - % self.env_id) -diff --git a/deploy/deploy.py b/deploy/deploy.py -index 7648baf..ee3cb7a 100755 ---- a/deploy/deploy.py -+++ b/deploy/deploy.py -@@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter - from dha import DeploymentHardwareAdapter - from install_fuel_master import InstallFuelMaster - from deploy_env import CloudDeploy -+from deploy_cache import DeployCache - from execution_environment import ExecutionEnvironment - - from common import ( -@@ -61,7 +62,8 @@ class AutoDeploy(object): - def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only, - cleanup, storage_dir, pxe_bridge, iso_file, dea_file, - dha_file, fuel_plugins_dir, fuel_plugins_conf_dir, -- no_plugins, deploy_timeout, no_deploy_environment, deploy_log): -+ no_plugins, deploy_cache_dir, deploy_timeout, -+ no_deploy_environment, deploy_log): - self.no_fuel = no_fuel - self.fuel_only = fuel_only - self.no_health_check = no_health_check -@@ -75,6 +77,7 @@ class AutoDeploy(object): - self.fuel_plugins_dir = fuel_plugins_dir - self.fuel_plugins_conf_dir = fuel_plugins_conf_dir - self.no_plugins = no_plugins -+ self.deploy_cache_dir = deploy_cache_dir - self.deploy_timeout = deploy_timeout - self.no_deploy_environment = no_deploy_environment - self.deploy_log = deploy_log -@@ -116,7 +119,7 @@ class AutoDeploy(object): - self.fuel_username, self.fuel_password, - self.dea_file, self.fuel_plugins_conf_dir, - WORK_DIR, self.no_health_check, -- self.deploy_timeout, -+ self.deploy_cache_dir, self.deploy_timeout, - self.no_deploy_environment, self.deploy_log) - with old_dep.ssh: - old_dep.check_previous_installation() -@@ -128,6 +131,7 @@ class AutoDeploy(object): - self.fuel_conf['ip'], self.fuel_username, - self.fuel_password, self.fuel_node_id, - self.iso_file, WORK_DIR, -+ self.deploy_cache_dir, - self.fuel_plugins_dir, self.no_plugins) - fuel.install() - -@@ -136,6 +140,7 @@ class AutoDeploy(object): - tmp_new_dir = '%s/newiso' % self.tmp_dir - try: - self.copy(tmp_orig_dir, tmp_new_dir) -+ self.deploy_cache_fingerprints(tmp_new_dir) - self.patch(tmp_new_dir, new_iso) - except Exception as e: - exec_cmd('fusermount -u %s' % tmp_orig_dir, False) -@@ -156,6 +161,12 @@ class AutoDeploy(object): - delete(tmp_orig_dir) - exec_cmd('chmod -R 755 %s' % tmp_new_dir) - -+ def deploy_cache_fingerprints(self, tmp_new_dir): -+ if self.deploy_cache_dir: -+ log('Deploy cache: Collecting fingerprints...') -+ deploy_cache = DeployCache(self.deploy_cache_dir) -+ deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file) -+ - def patch(self, tmp_new_dir, new_iso): - log('Patching...') - patch_dir = '%s/%s' % (CWD, PATCH_DIR) -@@ -218,7 +229,8 @@ class AutoDeploy(object): - dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'], - self.fuel_username, self.fuel_password, - self.dea_file, self.fuel_plugins_conf_dir, -- WORK_DIR, self.no_health_check, self.deploy_timeout, -+ WORK_DIR, self.no_health_check, -+ self.deploy_cache_dir, self.deploy_timeout, - self.no_deploy_environment, self.deploy_log) - return dep.deploy() - -@@ -343,6 +355,8 @@ def parse_arguments(): - help='Fuel Plugins Configuration directory') - parser.add_argument('-np', dest='no_plugins', action='store_true', - default=False, help='Do not install Fuel Plugins') -+ parser.add_argument('-dc', dest='deploy_cache_dir', action='store', -+ help='Deploy Cache Directory') - parser.add_argument('-dt', dest='deploy_timeout', action='store', - default=240, help='Deployment timeout (in minutes) ' - '[default: 240]') -@@ -376,6 +390,10 @@ def parse_arguments(): - for bridge in args.pxe_bridge: - check_bridge(bridge, args.dha_file) - -+ if args.deploy_cache_dir: -+ log('Using deploy cache directory: %s' % args.deploy_cache_dir) -+ create_dir_if_not_exists(args.deploy_cache_dir) -+ - - kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only, - 'no_health_check': args.no_health_check, -@@ -386,6 +404,7 @@ def parse_arguments(): - 'fuel_plugins_dir': args.fuel_plugins_dir, - 'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir, - 'no_plugins': args.no_plugins, -+ 'deploy_cache_dir': args.deploy_cache_dir, - 'deploy_timeout': args.deploy_timeout, - 'no_deploy_environment': args.no_deploy_environment, - 'deploy_log': args.deploy_log} -diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py -new file mode 100644 -index 0000000..30bfe30 ---- /dev/null -+++ b/deploy/deploy_cache.py -@@ -0,0 +1,314 @@ -+############################################################################### -+# Copyright (c) 2016 Enea AB and others. -+# Alexandru.Avadanii@enea.com -+# All rights reserved. This program and the accompanying materials -+# are made available under the terms of the Apache License, Version 2.0 -+# which accompanies this distribution, and is available at -+# http://www.apache.org/licenses/LICENSE-2.0 -+############################################################################### -+ -+import glob -+import hashlib -+import io -+import json -+import os -+import shutil -+import yaml -+ -+from common import ( -+ exec_cmd, -+ log, -+) -+ -+############################################################################### -+# Deploy Cache Flow Overview -+############################################################################### -+# 1. do_fingerprints -+# Can be called as soon as a Fuel Master ISO chroot is available. -+# This will gather all required information for uniquely identifying the -+# objects in cache (bootstraps, targetimages). -+# 2. inject_cache -+# Can be called as soon as we have a steady SSH connection to the Fuel -+# Master node. It will inject cached artifacts over SSH, for later install. -+# 3. (external, async) install cached bootstrap instead of building a new one -+# /sbin/bootstrap_admin_node.sh will check for cached bootstrap images -+# (with id_rsa, id_rsa.pub attached) and will install those via -+# $ fuel-bootstrap import opfnv-bootstraps-cache.tar -+# 4. install_targetimages_for_env -+# Should be called before cloud deploy is started, to install env-generic -+# 'env_X_...' cached images for the current environment ID. -+# Static method, to be used on the remote Fuel Master node; does not require -+# access to the deploy cache, it only moves around some local files. -+# 5. extract_targetimages_from_env -+# Should be called at env deploy finish, to prepare artifacts for caching. -+# Static method, same observations as above apply. -+# 6. collect_artifacts -+# Call last, to collect all artifacts. -+############################################################################### -+ -+############################################################################### -+# Deploy cache artifacts: -+# - id_rsa -+# - bootstrap image (Ubuntu) -+# - environment target image (Ubuntu) -+############################################################################### -+# Cache fingerprint covers: -+# - bootstrap: -+# - local mirror contents -+# - FIXME(armband): [disabled] package list (old fuel_bootstrap_cli.yaml) -+# - target image: -+# - local mirror contents -+# - package list (determined from DEA) -+############################################################################### -+# WARN: Cache fingerprint does NOT yet cover: -+# - image_data (always assume the default /boot, /); -+# - output_dir (always assume the default /var/www/nailgun/targetimages; -+# - codename (always assume the default, currently 'trusty'); -+# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty -+# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub -+# - Auxiliary repo .../mitaka-9.0/ubuntu/auxiliary -+# If the above change without triggering a cache miss, try clearing the cache. -+############################################################################### -+# WARN: Bootstrap caching implies RSA keypair to be reused! -+############################################################################### -+ -+# Local mirrros will be used on Fuel Master for both bootstrap and target image -+# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`: -+# - MOS .../mitaka-9.0/ubuntu/x86_64 -+# - Ubuntu .../mirrors/ubuntu/ -+# All these reside on Fuel Master at local path: -+NAILGUN_PATH = '/var/www/nailgun/' -+ -+# Artifact names (corresponding to nailgun subdirs) -+MIRRORS = 'mirrors' -+BOOTSTRAPS = 'bootstraps' -+TARGETIMAGES = 'targetimages' -+ -+# Info for collecting RSA keypair -+RSA_KEYPAIR_PATH = '/root/.ssh' -+RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub'] -+ -+# Relative path for collecting the active bootstrap image(s) after env deploy -+NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS -+ -+# Relative path for collecting target image(s) for deployed enviroment -+NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES -+ -+# FIXME(armband): re-include package list (old fuel_bootstrap_cli.yaml) -+# ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml' -+ -+# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected -+REMOTE_CACHE_PATH = '/var/cache/opnfv' -+ -+# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh -+BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar' -+ -+# Env-ID indep prefix -+ENVX = 'env_X_' -+ -+class DeployCache(object): -+ """OPNFV Deploy Cache - managed storage for cacheable artifacts""" -+ -+ def __init__(self, cache_dir, -+ fingerprints_yaml='deploy_cache_fingerprints.yaml'): -+ self.cache_dir = cache_dir -+ self.fingerprints_yaml = fingerprints_yaml -+ self.fingerprints = {BOOTSTRAPS: None, -+ MIRRORS: None, -+ TARGETIMAGES: None} -+ -+ def __load_fingerprints(self): -+ """Load deploy cache yaml config holding fingerprints""" -+ if os.path.isfile(self.fingerprints_yaml): -+ cache_fingerprints = open(self.fingerprints_yaml).read() -+ self.fingerprints = yaml.load(cache_fingerprints) -+ -+ def __save_fingerprints(self): -+ """Update deploy cache yaml config holding fingerprints""" -+ with open(self.fingerprints_yaml, 'w') as outfile: -+ outfile.write(yaml.safe_dump(self.fingerprints, -+ default_flow_style=False)) -+ -+ def __fingerprint_mirrors(self, chroot_path): -+ """Collect repo mirror fingerprints""" -+ deb_packages = list() -+ # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins) -+ for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']: -+ for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)): -+ for fdeb in files: -+ if fdeb.endswith(".deb"): -+ deb_packages.append(fdeb) -+ sorted_debs = json.dumps(deb_packages, sort_keys=True) -+ self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest() -+ -+ def __fingerprint_bootstrap(self, chroot_path): -+ """Collect bootstrap image metadata fingerprints""" -+ # FIXME(armband): include 'extra_dirs' contents -+ sorted_data = '' -+ # FIXME(armband): re-include package list (old fuel_bootstrap_cli.yaml) -+ # cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:]) -+ # bootstrap_cli_yaml = open(cli_yaml_path).read() -+ # bootstrap_data = yaml.load(bootstrap_cli_yaml) -+ # sorted_data = json.dumps(bootstrap_data, sort_keys=True) -+ self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest() -+ -+ def __fingerprint_target(self, dea_file): -+ """Collect target image metadata fingerprints""" -+ # FIXME(armband): include 'image_data', 'codename', 'output' -+ with io.open(dea_file) as stream: -+ dea = yaml.load(stream) -+ editable = dea['settings']['editable'] -+ target_data = {'packages': editable['provision']['packages'], -+ 'repos': editable['repo_setup']['repos']} -+ s_data = json.dumps(target_data, sort_keys=True) -+ self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest() -+ -+ def do_fingerprints(self, chroot_path, dea_file): -+ """Collect SHA1 fingerprints based on chroot contents, DEA settings""" -+ try: -+ self.__load_fingerprints() -+ self.__fingerprint_mirrors(chroot_path) -+ self.__fingerprint_bootstrap(chroot_path) -+ self.__fingerprint_target(dea_file) -+ self.__save_fingerprints() -+ except Exception as ex: -+ log('Failed to get cache fingerprint: %s' % str(ex)) -+ -+ def __lookup_cache(self, sha): -+ """Search for object in cache based on SHA fingerprint""" -+ cache_sha_dir = os.path.join(self.cache_dir, sha) -+ if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir): -+ return None -+ return cache_sha_dir -+ -+ def __inject_cache_dir(self, ssh, sha, artifact): -+ """Stage cached object (dir) in Fuel Master OPNFV local cache""" -+ local_path = self.__lookup_cache(sha) -+ if local_path: -+ remote_path = os.path.join(REMOTE_CACHE_PATH, artifact) -+ with ssh: -+ ssh.exec_cmd('mkdir -p %s' % remote_path) -+ for cachedfile in glob.glob('%s/*' % local_path): -+ ssh.scp_put(cachedfile, remote_path) -+ return local_path -+ -+ def __mix_fingerprints(self, f1, f2): -+ """Compute composite fingerprint""" -+ if self.fingerprints[f1] is None or self.fingerprints[f2] is None: -+ return None -+ return hashlib.sha1('%s%s' % -+ (self.fingerprints[f1], self.fingerprints[f2])).hexdigest() -+ -+ def inject_cache(self, ssh): -+ """Lookup artifacts in cache and inject them over SSH/SCP into Fuel""" -+ try: -+ self.__load_fingerprints() -+ for artifact in [BOOTSTRAPS, TARGETIMAGES]: -+ sha = self.__mix_fingerprints(MIRRORS, artifact) -+ if sha is None: -+ log('Missing fingerprint for: %s' % artifact) -+ continue -+ if not self.__inject_cache_dir(ssh, sha, artifact): -+ log('SHA1 not in cache: %s (%s)' % (str(sha), artifact)) -+ else: -+ log('SHA1 injected: %s (%s)' % (str(sha), artifact)) -+ except Exception as ex: -+ log('Failed to inject cached artifacts into Fuel: %s' % str(ex)) -+ -+ def __extract_bootstraps(self, ssh, cache_sha_dir): -+ """Collect bootstrap artifacts from Fuel over SSH/SCP""" -+ remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE) -+ local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE) -+ with ssh: -+ for k in RSA_KEYPAIR_FILES: -+ ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k), -+ local=os.path.join(cache_sha_dir, k)) -+ ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' % -+ (REMOTE_CACHE_PATH, -+ os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR), -+ remote_tar)) -+ ssh.scp_get(remote_tar, local=local_tar) -+ ssh.exec_cmd('rm -f %s' % remote_tar) -+ -+ def __extract_targetimages(self, ssh, cache_sha_dir): -+ """Collect target image artifacts from Fuel over SSH/SCP""" -+ cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) -+ with ssh: -+ ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir) -+ -+ def collect_artifacts(self, ssh): -+ """Collect artifacts from Fuel over SSH/SCP and add them to cache""" -+ try: -+ self.__load_fingerprints() -+ for artifact, func in { -+ BOOTSTRAPS: self.__extract_bootstraps, -+ TARGETIMAGES: self.__extract_targetimages -+ }.iteritems(): -+ sha = self.__mix_fingerprints(MIRRORS, artifact) -+ if sha is None: -+ log('WARN: Skip caching, NO fingerprint: %s' % artifact) -+ continue -+ local_path = self.__lookup_cache(sha) -+ if local_path: -+ log('SHA1 already in cache: %s (%s)' % (str(sha), artifact)) -+ else: -+ log('New cache SHA1: %s (%s)' % (str(sha), artifact)) -+ cache_sha_dir = os.path.join(self.cache_dir, sha) -+ exec_cmd('mkdir -p %s' % cache_sha_dir) -+ func(ssh, cache_sha_dir) -+ except Exception as ex: -+ log('Failed to extract artifacts from Fuel: %s' % str(ex)) -+ -+ @staticmethod -+ def extract_targetimages_from_env(env_id): -+ """Prepare targetimages from env ID for storage in deploy cache -+ -+ NOTE: This method should be executed locally ON the Fuel Master node. -+ WARN: This method overwrites targetimages cache on Fuel Master node. -+ """ -+ env_n = 'env_%s_' % str(env_id) -+ cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) -+ ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR) -+ try: -+ exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path)) -+ for root, _, files in os.walk(ti_path): -+ for tif in files: -+ if tif.startswith(env_n): -+ src = os.path.join(root, tif) -+ dest = os.path.join(cti_path, tif.replace(env_n, ENVX)) -+ if tif.endswith('.yaml'): -+ shutil.copy(src, dest) -+ exec_cmd('sed -i "s|%s|%s|g" %s' % -+ (env_n, ENVX, dest)) -+ else: -+ os.link(src, dest) -+ except Exception as ex: -+ log('Failed to extract targetimages artifacts from env %s: %s' % -+ (str(env_id), str(ex))) -+ -+ @staticmethod -+ def install_targetimages_for_env(env_id): -+ """Install targetimages artifacts for a specific env ID -+ -+ NOTE: This method should be executed locally ON the Fuel Master node. -+ """ -+ env_n = 'env_%s_' % str(env_id) -+ cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) -+ ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR) -+ if not os.path.isdir(cti_path): -+ log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path)) -+ else: -+ try: -+ for root, _, files in os.walk(cti_path): -+ for tif in files: -+ src = os.path.join(root, tif) -+ dest = os.path.join(ti_path, tif.replace(ENVX, env_n)) -+ if tif.endswith('.yaml'): -+ shutil.copy(src, dest) -+ exec_cmd('sed -i "s|%s|%s|g" %s' % -+ (ENVX, env_n, dest)) -+ else: -+ os.link(src, dest) -+ except Exception as ex: -+ log('Failed to install targetimages for env %s: %s' % -+ (str(env_id), str(ex))) -diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py -index d374cce..445070a 100644 ---- a/deploy/deploy_env.py -+++ b/deploy/deploy_env.py -@@ -15,6 +15,7 @@ import glob - import time - import shutil - -+from deploy_cache import DeployCache - from ssh_client import SSHClient - - from common import ( -@@ -35,7 +36,8 @@ class CloudDeploy(object): - - def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password, - dea_file, fuel_plugins_conf_dir, work_dir, no_health_check, -- deploy_timeout, no_deploy_environment, deploy_log): -+ deploy_cache_dir, deploy_timeout, -+ no_deploy_environment, deploy_log): - self.dea = dea - self.dha = dha - self.fuel_ip = fuel_ip -@@ -49,6 +51,8 @@ class CloudDeploy(object): - self.fuel_plugins_conf_dir = fuel_plugins_conf_dir - self.work_dir = work_dir - self.no_health_check = no_health_check -+ self.deploy_cache = ( DeployCache(deploy_cache_dir) -+ if deploy_cache_dir else None ) - self.deploy_timeout = deploy_timeout - self.no_deploy_environment = no_deploy_environment - self.deploy_log = deploy_log -@@ -82,9 +86,14 @@ class CloudDeploy(object): - self.work_dir, os.path.basename(self.dea_file))) - s.scp_put('%s/common.py' % self.file_dir, self.work_dir) - s.scp_put('%s/dea.py' % self.file_dir, self.work_dir) -+ s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir) - for f in glob.glob('%s/cloud/*' % self.file_dir): - s.scp_put(f, self.work_dir) - -+ def deploy_cache_collect_artifacts(self): -+ if self.deploy_cache: -+ self.deploy_cache.collect_artifacts(self.ssh) -+ - def power_off_nodes(self): - for node_id in self.node_ids: - self.dha.node_power_off(node_id) -@@ -281,4 +290,6 @@ class CloudDeploy(object): - - self.get_put_deploy_log() - -+ self.deploy_cache_collect_artifacts() -+ - return rc -diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py -index b731c6b..83d31fb 100644 ---- a/deploy/install_fuel_master.py -+++ b/deploy/install_fuel_master.py -@@ -10,6 +10,7 @@ - import time - import os - import glob -+from deploy_cache import DeployCache - from ssh_client import SSHClient - from dha_adapters.libvirt_adapter import LibvirtAdapter - -@@ -32,7 +33,7 @@ class InstallFuelMaster(object): - - def __init__(self, dea_file, dha_file, fuel_ip, fuel_username, - fuel_password, fuel_node_id, iso_file, work_dir, -- fuel_plugins_dir, no_plugins): -+ deploy_cache_dir, fuel_plugins_dir, no_plugins): - self.dea_file = dea_file - self.dha = LibvirtAdapter(dha_file) - self.fuel_ip = fuel_ip -@@ -42,6 +43,8 @@ class InstallFuelMaster(object): - self.iso_file = iso_file - self.iso_dir = os.path.dirname(self.iso_file) - self.work_dir = work_dir -+ self.deploy_cache = ( DeployCache(deploy_cache_dir) -+ if deploy_cache_dir else None ) - self.fuel_plugins_dir = fuel_plugins_dir - self.no_plugins = no_plugins - self.file_dir = os.path.dirname(os.path.realpath(__file__)) -@@ -83,6 +86,10 @@ class InstallFuelMaster(object): - log('Wait until Fuel menu is up') - fuel_menu_pid = self.wait_until_fuel_menu_up() - -+ if self.deploy_cache: -+ log('Deploy cache: Injecting bootstraps and targetimages') -+ self.deploy_cache.inject_cache(self.ssh) -+ - log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings') - self.inject_own_astute_and_bootstrap_yaml() - |