From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Date: Thu, 24 Nov 2016 23:02:04 +0100
Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts

Add support for caching deploy artifacts, like bootstraps and
target images, which take a lot of time at each deploy to be built,
considering it requires a cross-debootstrap via qemu-user-static and
binfmt.

For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism,
and be located at:
/iso_mount/opnfv_ci/<branch>/deploy-cache

TODO: Use dea interface adapter in target images fingerprinting.
TODO: remote fingerprinting
TODO: differentiate between bootstraps and targetimages, so we don't
end up trying to use one cache artifact type as the other.
TODO: implement sanity checks for bootstrap and target images;
TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`;

JIRA: ARMBAND-172

Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---
 ...p_admin_node.sh-deploy_cache-install-hook.patch |  69 +++++
 ci/deploy.sh                                       |  14 +-
 deploy/cloud/deployment.py                         |  12 +
 deploy/deploy.py                                   |  25 +-
 deploy/deploy_cache.py                             | 321 +++++++++++++++++++++
 deploy/deploy_env.py                               |  13 +-
 deploy/install_fuel_master.py                      |   9 +-
 7 files changed, 454 insertions(+), 9 deletions(-)
 create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
 create mode 100644 deploy/deploy_cache.py

diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
new file mode 100644
index 0000000..80cd0f4
--- /dev/null
+++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
@@ -0,0 +1,69 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Mon, 28 Nov 2016 14:27:48 +0100
+Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
+
+Tooling on the automatic deploy side was updated to support deploy
+caching of artifacts like bootstrap (and id_rsa keypair), target
+images etc.
+
+Add installation hook that calls `fuel-bootstrap import` instead of
+`build` when a bootstrap tar is available in the agreed location,
+/var/lib/opnfv/cache/bootstraps/.
+
+JIRA: ARMBAND-172
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ iso/bootstrap_admin_node.sh | 20 +++++++++++++++++++-
+ 1 file changed, 19 insertions(+), 1 deletion(-)
+
+diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
+index abc5ffb..15e6261 100755
+--- a/iso/bootstrap_admin_node.sh
++++ b/iso/bootstrap_admin_node.sh
+@@ -61,6 +61,8 @@ wget \
+
+ ASTUTE_YAML='/etc/fuel/astute.yaml'
+ BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps"
++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
+ bs_build_log='/var/log/fuel-bootstrap-image-build.log'
+ bs_status=0
+ # Backup network configs to this folder. Folder will be created only if
+@@ -94,6 +96,7 @@ image becomes available, reboot nodes that failed to be discovered."
+ bs_done_message="Default bootstrap image building done. Now you can boot new \
+ nodes over PXE, they will be discovered and become available for installing \
+ OpenStack on them"
++bs_cache_message="OPNFV deploy cache: bootstrap image injected."
+ # Update issues messages
+ update_warn_message="There is an issue connecting to update repository of \
+ your distributions of OpenStack. \
+@@ -500,12 +503,27 @@ set_ui_bootstrap_error () {
+ 	EOF
+ }
+
++function inject_cached_ubuntu_bootstrap () {
++        if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
++             -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
++             -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
++          if cp "${OPNFV_CACHE_PATH}/id_rsa*" "~/.ssh/" && \
++                fuel-bootstrap -v --debug import --activate \
++                "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
++            fuel notify --topic "done" --send "${bs_cache_message}"
++            return 0
++          fi
++        fi
++        return 1
++}
++
+ # Actually build the bootstrap image
+ build_ubuntu_bootstrap () {
+         local ret=1
+         echo ${bs_progress_message} >&2
+         set_ui_bootstrap_error "${bs_progress_message}" >&2
+-        if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then
++        if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \
++          build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then
+           ret=0
+           fuel notify --topic "done" --send "${bs_done_message}"
+         else
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 081806c..4b1ae0e 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -29,7 +29,7 @@ cat << EOF
 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 `basename $0`: Deploys the Fuel@OPNFV stack

-usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso
+usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso
        -s deployment-scenario [-S optional Deploy-scenario path URI]
        [-R optional local relen repo (containing deployment Scenarios]

@@ -47,6 +47,7 @@ OPTIONS:
   -p  Pod-name
   -s  Deploy-scenario short-name/base-file-name
   -S  Storage dir for VM images
+  -C  Deploy cache dir for storing image artifacts
   -T  Timeout, in minutes, for the deploy.
   -i  iso url

@@ -79,6 +80,7 @@ Input parameters to the build script is:
    or a deployment short-name as defined by scenario.yaml in the deployment
    scenario path.
 -S Storage dir for VM images, default is fuel/deploy/images
+-C Deploy cache dir for bootstrap and target image artifacts, optional
 -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT
    environment variable when defined, or to the default in deploy.py otherwise
 -i .iso image to be deployed (needs to be provided in a URI
@@ -116,6 +118,7 @@ FUEL_CREATION_ONLY=''
 NO_DEPLOY_ENVIRONMENT=''
 STORAGE_DIR=''
 DRY_RUN=0
+DEPLOY_CACHE_DIR=''
 if ! [ -z $DEPLOY_TIMEOUT ]; then
     DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT"
 else
@@ -128,7 +131,7 @@ fi
 ############################################################################
 # BEGIN of main
 #
-while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION
+while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION
 do
     case $OPTION in
         b)
@@ -179,6 +182,9 @@ do
                 STORAGE_DIR="-s ${OPTARG}"
             fi
             ;;
+        C)
+            DEPLOY_CACHE_DIR="-dc ${OPTARG}"
+            ;;
         T)
             DEPLOY_TIMEOUT="-dt ${OPTARG}"
             ;;
@@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then
         ISO=${SCRIPT_PATH}/ISO/image.iso
     fi
     # Start deployment
-    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT"
-    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT
+    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR"
+    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR
 fi
 popd > /dev/null

diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
index 5dd0263..3db4c0d 100644
--- a/deploy/cloud/deployment.py
+++ b/deploy/cloud/deployment.py
@@ -24,6 +24,8 @@ from common import (
     delete,
 )

+from deploy_cache import DeployCache
+
 SEARCH_TEXT = '(err)'
 LOG_FILE = '/var/log/puppet.log'
 GREP_LINES_OF_LEADING_CONTEXT = 100
@@ -52,6 +54,14 @@ class Deployment(object):
         self.pattern = re.compile(
             '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')

+    def deploy_cache_install_targetimages(self):
+        log('Using target images from deploy cache')
+        DeployCache.install_targetimages_for_env(self.env_id)
+
+    def deploy_cache_extract_targetimages(self):
+        log('Collecting Fuel target image files for deploy cache')
+        DeployCache.extract_targetimages_from_env(self.env_id)
+
     def collect_error_logs(self):
         for node_id, roles_blade in self.node_id_roles_dict.iteritems():
             log_list = []
@@ -113,6 +123,7 @@ class Deployment(object):
         start = time.time()

         log('Starting deployment of environment %s' % self.env_id)
+        self.deploy_cache_install_targetimages()
         deploy_id = None
         ready = False
         timeout = False
@@ -145,6 +156,7 @@ class Deployment(object):
             err('Deployment timed out, environment %s is not operational, '
                 'snapshot will not be performed'
                 % self.env_id)
+        self.deploy_cache_extract_targetimages()
         if ready:
             log('Environment %s successfully deployed'
                 % self.env_id)
diff --git a/deploy/deploy.py b/deploy/deploy.py
index 08702d2..1a55361 100755
--- a/deploy/deploy.py
+++ b/deploy/deploy.py
@@ -23,6 +23,7 @@ from dea import DeploymentEnvironmentAdapter
 from dha import DeploymentHardwareAdapter
 from install_fuel_master import InstallFuelMaster
 from deploy_env import CloudDeploy
+from deploy_cache import DeployCache
 from execution_environment import ExecutionEnvironment

 from common import (
@@ -62,7 +63,8 @@ class AutoDeploy(object):
     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
-                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
+                 no_plugins, deploy_cache_dir, deploy_timeout,
+                 no_deploy_environment, deploy_log):
         self.no_fuel = no_fuel
         self.fuel_only = fuel_only
         self.no_health_check = no_health_check
@@ -76,6 +78,7 @@ class AutoDeploy(object):
         self.fuel_plugins_dir = fuel_plugins_dir
         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
         self.no_plugins = no_plugins
+        self.deploy_cache_dir = deploy_cache_dir
         self.deploy_timeout = deploy_timeout
         self.no_deploy_environment = no_deploy_environment
         self.deploy_log = deploy_log
@@ -117,7 +120,7 @@ class AutoDeploy(object):
                                   self.fuel_username, self.fuel_password,
                                   self.dea_file, self.fuel_plugins_conf_dir,
                                   WORK_DIR, self.no_health_check,
-                                  self.deploy_timeout,
+                                  self.deploy_cache_dir, self.deploy_timeout,
                                   self.no_deploy_environment, self.deploy_log)
             with old_dep.ssh:
                 old_dep.check_previous_installation()
@@ -129,6 +132,7 @@ class AutoDeploy(object):
                                  self.fuel_conf['ip'], self.fuel_username,
                                  self.fuel_password, self.fuel_node_id,
                                  self.iso_file, WORK_DIR,
+                                 self.deploy_cache_dir,
                                  self.fuel_plugins_dir, self.no_plugins)
         fuel.install()

@@ -137,6 +141,7 @@ class AutoDeploy(object):
         tmp_new_dir = '%s/newiso' % self.tmp_dir
         try:
             self.copy(tmp_orig_dir, tmp_new_dir)
+            self.deploy_cache_fingerprints(tmp_new_dir)
             self.patch(tmp_new_dir, new_iso)
         except Exception as e:
             exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
@@ -157,6 +162,12 @@ class AutoDeploy(object):
         delete(tmp_orig_dir)
         exec_cmd('chmod -R 755 %s' % tmp_new_dir)

+    def deploy_cache_fingerprints(self, tmp_new_dir):
+        if self.deploy_cache_dir:
+            log('Deploy cache: Collecting fingerprints...')
+            deploy_cache = DeployCache(self.deploy_cache_dir)
+            deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file)
+
     def patch(self, tmp_new_dir, new_iso):
         log('Patching...')
         patch_dir = '%s/%s' % (CWD, PATCH_DIR)
@@ -219,7 +230,8 @@ class AutoDeploy(object):
         dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
                           self.fuel_username, self.fuel_password,
                           self.dea_file, self.fuel_plugins_conf_dir,
-                          WORK_DIR, self.no_health_check, self.deploy_timeout,
+                          WORK_DIR, self.no_health_check,
+                          self.deploy_cache_dir, self.deploy_timeout,
                           self.no_deploy_environment, self.deploy_log)
         return dep.deploy()

@@ -344,6 +356,8 @@ def parse_arguments():
                         help='Fuel Plugins Configuration directory')
     parser.add_argument('-np', dest='no_plugins', action='store_true',
                         default=False, help='Do not install Fuel Plugins')
+    parser.add_argument('-dc', dest='deploy_cache_dir', action='store',
+                        help='Deploy Cache Directory')
     parser.add_argument('-dt', dest='deploy_timeout', action='store',
                         default=240, help='Deployment timeout (in minutes) '
                         '[default: 240]')
@@ -377,6 +391,10 @@ def parse_arguments():
         for bridge in args.pxe_bridge:
             check_bridge(bridge, args.dha_file)

+    if args.deploy_cache_dir:
+        log('Using deploy cache directory: %s' % args.deploy_cache_dir)
+        create_dir_if_not_exists(args.deploy_cache_dir)
+

     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
               'no_health_check': args.no_health_check,
@@ -387,6 +405,7 @@ def parse_arguments():
               'fuel_plugins_dir': args.fuel_plugins_dir,
               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
               'no_plugins': args.no_plugins,
+              'deploy_cache_dir': args.deploy_cache_dir,
               'deploy_timeout': args.deploy_timeout,
               'no_deploy_environment': args.no_deploy_environment,
               'deploy_log': args.deploy_log}
diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
new file mode 100644
index 0000000..76fb1b9
--- /dev/null
+++ b/deploy/deploy_cache.py
@@ -0,0 +1,321 @@
+###############################################################################
+# Copyright (c) 2016 Enea AB and others.
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+import glob
+import hashlib
+import io
+import json
+import os
+import shutil
+import yaml
+
+from common import (
+    exec_cmd,
+    log,
+)
+
+###############################################################################
+# Deploy Cache Flow Overview
+###############################################################################
+# 1. do_fingerprints
+#    Can be called as soon as a Fuel Master ISO chroot is available.
+#    This will gather all required information for uniquely identifying the
+#    objects in cache (bootstraps, targetimages).
+# 2. inject_cache
+#    Can be called as soon as we have a steady SSH connection to the Fuel
+#    Master node. It will inject cached artifacts over SSH, for later install.
+# 3. (external, async) install cached bootstrap instead of building a new one
+#    /sbin/bootstrap_admin_node.sh will check for cached bootstrap images
+#    (with id_rsa, id_rsa.pub attached) and will install those via
+#    $ fuel-bootstrap import opfnv-bootstraps-cache.tar
+# 4. install_targetimages_for_env
+#    Should be called before cloud deploy is started, to install env-generic
+#    'env_X_...' cached images for the current environment ID.
+#    Static method, to be used on the remote Fuel Master node; does not require
+#    access to the deploy cache, it only moves around some local files.
+# 5. extract_targetimages_from_env
+#    Should be called at env deploy finish, to prepare artifacts for caching.
+#    Static method, same observations as above apply.
+# 6. collect_artifacts
+#    Call last, to collect all artifacts.
+###############################################################################
+
+###############################################################################
+# Deploy cache artifacts:
+# - id_rsa
+# - bootstrap image (Ubuntu)
+# - environment target image (Ubuntu)
+###############################################################################
+# Cache fingerprint covers:
+# - bootstrap:
+#   - local mirror contents
+#   - package list (and everything else in fuel_bootstrap_cli.yaml)
+# - target image:
+#   - local mirror contents
+#   - package list (determined from DEA)
+###############################################################################
+# WARN: Cache fingerprint does NOT yet cover:
+# - image_data (always assume the default /boot, /);
+# - output_dir (always assume the default /var/www/nailgun/targetimages;
+# - codename (always assume the default, currently 'trusty');
+# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty
+# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub
+# - Auxiliary repo  .../mitaka-9.0/ubuntu/auxiliary
+# If the above change without triggering a cache miss, try clearing the cache.
+###############################################################################
+# WARN: Bootstrap caching implies RSA keypair to be reused!
+###############################################################################
+
+# Local mirrros will be used on Fuel Master for both bootstrap and target image
+# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`:
+# - MOS        .../mitaka-9.0/ubuntu/x86_64
+# - Ubuntu     .../mirrors/ubuntu/
+# All these reside on Fuel Master at local path:
+NAILGUN_PATH = '/var/www/nailgun/'
+
+# Artifact names (corresponding to nailgun subdirs)
+MIRRORS = 'mirrors'
+BOOTSTRAPS = 'bootstraps'
+TARGETIMAGES = 'targetimages'
+
+# Info for collecting RSA keypair
+RSA_KEYPAIR_PATH = '/root/.ssh'
+RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub']
+
+# Relative path for collecting the active bootstrap image(s) after env deploy
+NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS
+
+# Relative path for collecting target image(s) for deployed enviroment
+NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES
+
+# OPNFV Fuel bootstrap settings file that will be injected at deploy
+ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
+
+# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
+REMOTE_CACHE_PATH = '/var/cache/opnfv'
+
+# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
+BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
+
+# Env-ID indep prefix
+ENVX = 'env_X_'
+
+class DeployCache(object):
+    """OPNFV Deploy Cache - managed storage for cacheable artifacts"""
+
+    def __init__(self, cache_dir,
+                 fingerprints_yaml='deploy_cache_fingerprints.yaml'):
+        self.cache_dir = cache_dir
+        self.fingerprints_yaml = fingerprints_yaml
+        self.fingerprints = {BOOTSTRAPS: None,
+                             MIRRORS: None,
+                             TARGETIMAGES: None}
+
+    def __load_fingerprints(self):
+        """Load deploy cache yaml config holding fingerprints"""
+        if os.path.isfile(self.fingerprints_yaml):
+            cache_fingerprints = open(self.fingerprints_yaml).read()
+            self.fingerprints = yaml.load(cache_fingerprints)
+
+    def __save_fingerprints(self):
+        """Update deploy cache yaml config holding fingerprints"""
+        with open(self.fingerprints_yaml, 'w') as outfile:
+            outfile.write(yaml.safe_dump(self.fingerprints,
+                          default_flow_style=False))
+
+    def __fingerprint_mirrors(self, chroot_path):
+        """Collect repo mirror fingerprints"""
+        md5sums = list()
+        # Scan all ISO for deb repo metadata and collect MD5 from Release files
+        for root, _, files in os.walk(chroot_path):
+            for relf in files:
+                if relf == 'Release' and 'binary' not in root:
+                    collect_sums = False
+                    filepath = os.path.join(root, relf)
+                    with open(filepath, "r") as release_file:
+                        for line in release_file:
+                            if collect_sums:
+                                if line.startswith(' '):
+                                    md5sums += [line[1:33]]
+                                else:
+                                    break
+                            elif line.startswith('MD5Sum:'):
+                                collect_sums = True
+        sorted_md5sums = json.dumps(md5sums, sort_keys=True)
+        self.fingerprints[MIRRORS] = hashlib.sha1(sorted_md5sums).hexdigest()
+
+    def __fingerprint_bootstrap(self, chroot_path):
+        """Collect bootstrap image metadata fingerprints"""
+        # FIXME(armband): include 'extra_dirs' contents
+        cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:])
+        bootstrap_cli_yaml = open(cli_yaml_path).read()
+        bootstrap_data = yaml.load(bootstrap_cli_yaml)
+        sorted_data = json.dumps(bootstrap_data, sort_keys=True)
+        self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest()
+
+    def __fingerprint_target(self, dea_file):
+        """Collect target image metadata fingerprints"""
+        # FIXME(armband): include 'image_data', 'codename', 'output'
+        with io.open(dea_file) as stream:
+            dea = yaml.load(stream)
+            editable = dea['settings']['editable']
+            target_data = {'packages': editable['provision']['packages'],
+                           'repos': editable['repo_setup']['repos']}
+            s_data = json.dumps(target_data, sort_keys=True)
+            self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest()
+
+    def do_fingerprints(self, chroot_path, dea_file):
+        """Collect SHA1 fingerprints based on chroot contents, DEA settings"""
+        try:
+            self.__load_fingerprints()
+            self.__fingerprint_mirrors(chroot_path)
+            self.__fingerprint_bootstrap(chroot_path)
+            self.__fingerprint_target(dea_file)
+            self.__save_fingerprints()
+        except Exception as ex:
+            log('Failed to get cache fingerprint: %s' % str(ex))
+
+    def __lookup_cache(self, sha):
+        """Search for object in cache based on SHA fingerprint"""
+        cache_sha_dir = os.path.join(self.cache_dir, sha)
+        if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir):
+            return None
+        return cache_sha_dir
+
+    def __inject_cache_dir(self, ssh, sha, artifact):
+        """Stage cached object (dir) in Fuel Master OPNFV local cache"""
+        local_path = self.__lookup_cache(sha)
+        if local_path:
+            remote_path = os.path.join(REMOTE_CACHE_PATH, artifact)
+            with ssh:
+                ssh.exec_cmd('mkdir -p %s' % remote_path)
+                for cachedfile in glob.glob('%s/*' % local_path):
+                    ssh.scp_put(cachedfile, remote_path)
+        return local_path
+
+    def __mix_fingerprints(self, f1, f2):
+        """Compute composite fingerprint"""
+        if self.fingerprints[f1] is None or self.fingerprints[f2] is None:
+            return None
+        return hashlib.sha1('%s%s' %
+            (self.fingerprints[f1], self.fingerprints[f2])).hexdigest()
+
+    def inject_cache(self, ssh):
+        """Lookup artifacts in cache and inject them over SSH/SCP into Fuel"""
+        try:
+            self.__load_fingerprints()
+            for artifact in [BOOTSTRAPS, TARGETIMAGES]:
+                sha = self.__mix_fingerprints(MIRRORS, artifact)
+                if sha is None:
+                    log('Missing fingerprint for: %s' % artifact)
+                    continue
+                if not self.__inject_cache_dir(ssh, sha, artifact):
+                    log('SHA1 not in cache: %s (%s)' % (str(sha), artifact))
+                else:
+                    log('SHA1 injected: %s (%s)' % (str(sha), artifact))
+        except Exception as ex:
+            log('Failed to inject cached artifacts into Fuel: %s' % str(ex))
+
+    def __extract_bootstraps(self, ssh, cache_sha_dir):
+        """Collect bootstrap artifacts from Fuel over SSH/SCP"""
+        remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE)
+        local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE)
+        with ssh:
+            for k in RSA_KEYPAIR_FILES:
+                ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
+                    local=os.path.join(cache_sha_dir, k))
+            ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' %
+                (REMOTE_CACHE_PATH,
+                os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR),
+                remote_tar))
+            ssh.scp_get(remote_tar, local=local_tar)
+            ssh.exec_cmd('rm -f %s' % remote_tar)
+
+    def __extract_targetimages(self, ssh, cache_sha_dir):
+        """Collect target image artifacts from Fuel over SSH/SCP"""
+        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
+        with ssh:
+            ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir)
+
+    def collect_artifacts(self, ssh):
+        """Collect artifacts from Fuel over SSH/SCP and add them to cache"""
+        try:
+            self.__load_fingerprints()
+            for artifact, func in {
+                    BOOTSTRAPS: self.__extract_bootstraps,
+                    TARGETIMAGES: self.__extract_targetimages
+                }.iteritems():
+                sha = self.__mix_fingerprints(MIRRORS, artifact)
+                if sha is None:
+                    log('WARN: Skip caching, NO fingerprint: %s' % artifact)
+                    continue
+                local_path = self.__lookup_cache(sha)
+                if local_path:
+                    log('SHA1 already in cache: %s (%s)' % (str(sha), artifact))
+                else:
+                    log('New cache SHA1: %s (%s)' % (str(sha), artifact))
+                    cache_sha_dir = os.path.join(self.cache_dir, sha)
+                    exec_cmd('mkdir -p %s' % cache_sha_dir)
+                    func(ssh, cache_sha_dir)
+        except Exception as ex:
+            log('Failed to extract artifacts from Fuel: %s' % str(ex))
+
+    @staticmethod
+    def extract_targetimages_from_env(env_id):
+        """Prepare targetimages from env ID for storage in deploy cache
+
+        NOTE: This method should be executed locally ON the Fuel Master node.
+        WARN: This method overwrites targetimages cache on Fuel Master node.
+        """
+        env_n = 'env_%s_' % str(env_id)
+        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
+        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
+        try:
+            exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path))
+            for root, _, files in os.walk(ti_path):
+                for tif in files:
+                    if tif.startswith(env_n):
+                        src = os.path.join(root, tif)
+                        dest = os.path.join(cti_path, tif.replace(env_n, ENVX))
+                        if tif.endswith('.yaml'):
+                            shutil.copy(src, dest)
+                            exec_cmd('sed -i "s|%s|%s|g" %s' %
+                                     (env_n, ENVX, dest))
+                        else:
+                            os.link(src, dest)
+        except Exception as ex:
+            log('Failed to extract targetimages artifacts from env %s: %s' %
+                (str(env_id), str(ex)))
+
+    @staticmethod
+    def install_targetimages_for_env(env_id):
+        """Install targetimages artifacts for a specific env ID
+
+        NOTE: This method should be executed locally ON the Fuel Master node.
+        """
+        env_n = 'env_%s_' % str(env_id)
+        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
+        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
+        if not os.path.isdir(cti_path):
+            log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path))
+        else:
+            try:
+                for root, _, files in os.walk(cti_path):
+                    for tif in files:
+                        src = os.path.join(root, tif)
+                        dest = os.path.join(ti_path, tif.replace(ENVX, env_n))
+                        if tif.endswith('.yaml'):
+                            shutil.copy(src, dest)
+                            exec_cmd('sed -i "s|%s|%s|g" %s' %
+                                     (ENVX, env_n, dest))
+                        else:
+                            os.link(src, dest)
+            except Exception as ex:
+                log('Failed to install targetimages for env %s: %s' %
+                    (str(env_id), str(ex)))
diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
index 1d2dfeb..2375f51 100644
--- a/deploy/deploy_env.py
+++ b/deploy/deploy_env.py
@@ -15,6 +15,7 @@ import glob
 import time
 import shutil

+from deploy_cache import DeployCache
 from ssh_client import SSHClient

 from common import (
@@ -36,7 +37,8 @@ class CloudDeploy(object):

     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
                  dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
-                 deploy_timeout, no_deploy_environment, deploy_log):
+                 deploy_cache_dir, deploy_timeout,
+                 no_deploy_environment, deploy_log):
         self.dea = dea
         self.dha = dha
         self.fuel_ip = fuel_ip
@@ -50,6 +52,8 @@ class CloudDeploy(object):
         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
         self.work_dir = work_dir
         self.no_health_check = no_health_check
+        self.deploy_cache = ( DeployCache(deploy_cache_dir)
+                              if deploy_cache_dir else None )
         self.deploy_timeout = deploy_timeout
         self.no_deploy_environment = no_deploy_environment
         self.deploy_log = deploy_log
@@ -83,9 +87,14 @@ class CloudDeploy(object):
                 self.work_dir, os.path.basename(self.dea_file)))
             s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
             s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
+            s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir)
             for f in glob.glob('%s/cloud/*' % self.file_dir):
                 s.scp_put(f, self.work_dir)

+    def deploy_cache_collect_artifacts(self):
+        if self.deploy_cache:
+            self.deploy_cache.collect_artifacts(self.ssh)
+
     def power_off_nodes(self):
         for node_id in self.node_ids:
             self.dha.node_power_off(node_id)
@@ -284,4 +293,6 @@ class CloudDeploy(object):

         self.get_put_deploy_log()

+        self.deploy_cache_collect_artifacts()
+
         return rc
diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
index ccc18d3..2615818 100644
--- a/deploy/install_fuel_master.py
+++ b/deploy/install_fuel_master.py
@@ -10,6 +10,7 @@
 import time
 import os
 import glob
+from deploy_cache import DeployCache
 from ssh_client import SSHClient
 from dha_adapters.libvirt_adapter import LibvirtAdapter

@@ -33,7 +34,7 @@ class InstallFuelMaster(object):

     def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
                  fuel_password, fuel_node_id, iso_file, work_dir,
-                 fuel_plugins_dir, no_plugins):
+                 deploy_cache_dir, fuel_plugins_dir, no_plugins):
         self.dea_file = dea_file
         self.dha = LibvirtAdapter(dha_file)
         self.fuel_ip = fuel_ip
@@ -43,6 +44,8 @@ class InstallFuelMaster(object):
         self.iso_file = iso_file
         self.iso_dir = os.path.dirname(self.iso_file)
         self.work_dir = work_dir
+        self.deploy_cache = ( DeployCache(deploy_cache_dir)
+                              if deploy_cache_dir else None )
         self.fuel_plugins_dir = fuel_plugins_dir
         self.no_plugins = no_plugins
         self.file_dir = os.path.dirname(os.path.realpath(__file__))
@@ -84,6 +87,10 @@ class InstallFuelMaster(object):
         log('Wait until Fuel menu is up')
         fuel_menu_pid = self.wait_until_fuel_menu_up()

+        if self.deploy_cache:
+            log('Deploy cache: Injecting bootstraps and targetimages')
+            self.deploy_cache.inject_cache(self.ssh)
+
         log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings')
         self.inject_own_astute_and_bootstrap_yaml()