summaryrefslogtreecommitdiffstats
path: root/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch
blob: 10a7f2c3695223446d83dabd6a19810d5a495b9f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
: Copyright (c) 2017 Enea AB and others.
:
: All rights reserved. This program and the accompanying materials
: are made available under the terms of the Apache License, Version 2.0
: which accompanies this distribution, and is available at
: http://www.apache.org/licenses/LICENSE-2.0
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Date: Thu, 24 Nov 2016 23:02:04 +0100
Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts

Add support for caching deploy artifacts, like bootstraps and
target images, which take a lot of time at each deploy to be built,
considering it requires a cross-debootstrap via qemu-user-static and
binfmt.

For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism,
and be located at:
/iso_mount/opnfv_ci/<branch>/deploy-cache

TODO: Use dea interface adapter in target images fingerprinting.
TODO: remote fingerprinting
TODO: differentiate between bootstraps and targetimages, so we don't
end up trying to use one cache artifact type as the other.
TODO: implement sanity checks for bootstrap and target images;
TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`;

JIRA: ARMBAND-172
JIRA: ARMBAND-242

Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---
 ...p_admin_node.sh-deploy_cache-install-hook.patch |  90 ++++++
 ci/deploy.sh                                       |  14 +-
 deploy/cloud/configure_settings.py                 |   4 +
 deploy/cloud/deployment.py                         |  12 +
 deploy/deploy.py                                   |  25 +-
 deploy/deploy_cache.py                             | 312 +++++++++++++++++++++
 deploy/deploy_env.py                               |  13 +-
 deploy/install_fuel_master.py                      |   9 +-
 8 files changed, 470 insertions(+), 9 deletions(-)
 create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
 create mode 100644 deploy/deploy_cache.py

diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
new file mode 100644
index 0000000..7acb746
--- /dev/null
+++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch
@@ -0,0 +1,90 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Mon, 28 Nov 2016 14:27:48 +0100
+Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook
+
+Tooling on the automatic deploy side was updated to support deploy
+caching of artifacts like bootstrap (and id_rsa keypair), target
+images etc.
+
+Add installation hook that calls `fuel-bootstrap import` instead of
+`build` when a bootstrap tar is available in the agreed location,
+/var/lib/opnfv/cache/bootstraps/.
+
+Temporary until Fuel@Openstack fixes Master key propagation to nodes'
+authorized_keys, use Mcollective remote shell execute to add it
+during deployment.
+This might duplicate the entry in authorized_keys during re-deploys.
+
+JIRA: ARMBAND-172
+JIRA: ARMBAND-242
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ iso/bootstrap_admin_node.sh | 35 ++++++++++++++++++++++++++++++++++-
+ 1 file changed, 34 insertions(+), 1 deletion(-)
+
+diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
+index 4f5ce4e..4c79552 100755
+--- a/iso/bootstrap_admin_node.sh
++++ b/iso/bootstrap_admin_node.sh
+@@ -64,6 +64,8 @@ wget \
+ ASTUTE_YAML='/etc/fuel/astute.yaml'
+ BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
+ CUSTOM_REPOS="/root/default_deb_repos.yaml"
++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps"
++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar"
+ bs_build_log='/var/log/fuel-bootstrap-image-build.log'
+ bs_status=0
+ # Backup network configs to this folder. Folder will be created only if
+@@ -97,6 +99,7 @@ image becomes available, reboot nodes that failed to be discovered."
+ bs_done_message="Default bootstrap image building done. Now you can boot new \
+ nodes over PXE, they will be discovered and become available for installing \
+ OpenStack on them"
++bs_cache_message="OPNFV deploy cache: bootstrap image injected."
+ # Update issues messages
+ update_warn_message="There is an issue connecting to update repository of \
+ your distributions of OpenStack. \
+@@ -509,12 +512,42 @@ set_ui_bootstrap_error () {
+ 	EOF
+ }
+
++function inject_cached_ssh_key () {
++	# FIXME(armband): Propagate master ssh key to nodes'
++	# authorized_keys, until upstream fixes this for image build.
++	local moddir="/etc/puppet/${OPENSTACK_VERSION}/modules/osnailyfacter/modular"
++	cat >> "${moddir}/astute/generate_keys.sh" <<-EOF
++	mco rpc execute_shell_command execute \\
++	cmd="echo $(cat /root/.ssh/id_rsa.pub) >> /root/.ssh/authorized_keys"
++	EOF
++}
++
++function inject_cached_ubuntu_bootstrap () {
++        if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \
++             -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \
++             -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then
++          if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \
++             cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \
++             cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \
++                sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \
++                /etc/nailgun/settings.yaml && \
++                fuel-bootstrap -v --debug import --activate \
++                "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then
++                    inject_cached_ssh_key
++                    fuel notify --topic "done" --send "${bs_cache_message}"
++                    return 0
++          fi
++        fi
++        return 1
++}
++
+ # Actually build the bootstrap image
+ build_ubuntu_bootstrap () {
+         local ret=1
+         echo ${bs_progress_message} >&2
+         set_ui_bootstrap_error "${bs_progress_message}" >&2
+-        if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then
++        if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \
++          build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then
+           ret=0
+           fuel notify --topic "done" --send "${bs_done_message}"
+         else
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 081806c..4b1ae0e 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -29,7 +29,7 @@ cat << EOF
 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 `basename $0`: Deploys the Fuel@OPNFV stack

-usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso
+usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso
        -s deployment-scenario [-S optional Deploy-scenario path URI]
        [-R optional local relen repo (containing deployment Scenarios]

@@ -47,6 +47,7 @@ OPTIONS:
   -p  Pod-name
   -s  Deploy-scenario short-name/base-file-name
   -S  Storage dir for VM images
+  -C  Deploy cache dir for storing image artifacts
   -T  Timeout, in minutes, for the deploy.
   -i  iso url

@@ -79,6 +80,7 @@ Input parameters to the build script is:
    or a deployment short-name as defined by scenario.yaml in the deployment
    scenario path.
 -S Storage dir for VM images, default is fuel/deploy/images
+-C Deploy cache dir for bootstrap and target image artifacts, optional
 -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT
    environment variable when defined, or to the default in deploy.py otherwise
 -i .iso image to be deployed (needs to be provided in a URI
@@ -116,6 +118,7 @@ FUEL_CREATION_ONLY=''
 NO_DEPLOY_ENVIRONMENT=''
 STORAGE_DIR=''
 DRY_RUN=0
+DEPLOY_CACHE_DIR=''
 if ! [ -z $DEPLOY_TIMEOUT ]; then
     DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT"
 else
@@ -128,7 +131,7 @@ fi
 ############################################################################
 # BEGIN of main
 #
-while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION
+while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION
 do
     case $OPTION in
         b)
@@ -179,6 +182,9 @@ do
                 STORAGE_DIR="-s ${OPTARG}"
             fi
             ;;
+        C)
+            DEPLOY_CACHE_DIR="-dc ${OPTARG}"
+            ;;
         T)
             DEPLOY_TIMEOUT="-dt ${OPTARG}"
             ;;
@@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then
         ISO=${SCRIPT_PATH}/ISO/image.iso
     fi
     # Start deployment
-    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT"
-    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT
+    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR"
+    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR
 fi
 popd > /dev/null

diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py
index b60a60f..4e007e1 100644
--- a/deploy/cloud/configure_settings.py
+++ b/deploy/cloud/configure_settings.py
@@ -71,5 +71,9 @@ class ConfigureSettings(object):
                         settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id']
                         settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id']

+        # deploy-cache req: pass master id_rsa.pub as authorized key
+        with io.open('/root/.ssh/id_rsa.pub', 'r') as pkey:
+            settings['editable']['operator_user']['authkeys']['value'] = pkey.read()
+
         with io.open(settings_yaml, 'w') as stream:
             yaml.dump(settings, stream, default_flow_style=False)
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
index 4329a4c..a84d46c 100644
--- a/deploy/cloud/deployment.py
+++ b/deploy/cloud/deployment.py
@@ -19,6 +19,8 @@ from common import (
     log,
 )

+from deploy_cache import DeployCache
+
 SEARCH_TEXT = '(err)'
 LOG_FILE = '/var/log/puppet.log'
 GREP_LINES_OF_LEADING_CONTEXT = 100
@@ -51,6 +53,14 @@ class Deployment(object):
         self.pattern = re.compile(
             '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')

+    def deploy_cache_install_targetimages(self):
+        log('Using target images from deploy cache')
+        DeployCache.install_targetimages_for_env(self.env_id)
+
+    def deploy_cache_extract_targetimages(self):
+        log('Collecting Fuel target image files for deploy cache')
+        DeployCache.extract_targetimages_from_env(self.env_id)
+
     def collect_error_logs(self):
         for node_id, roles_blade in self.node_id_roles_dict.iteritems():
             log_list = []
@@ -112,6 +122,7 @@ class Deployment(object):
         start = time.time()

         log('Starting deployment of environment %s' % self.env_id)
+        self.deploy_cache_install_targetimages()
         deploy_id = None
         ready = False
         timeout = False
@@ -144,6 +155,7 @@ class Deployment(object):
             err('Deployment timed out, environment %s is not operational, '
                 'snapshot will not be performed'
                 % self.env_id)
+        self.deploy_cache_extract_targetimages()
         if ready:
             log('Environment %s successfully deployed'
                 % self.env_id)
diff --git a/deploy/deploy.py b/deploy/deploy.py
index 7648baf..ee3cb7a 100755
--- a/deploy/deploy.py
+++ b/deploy/deploy.py
@@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter
 from dha import DeploymentHardwareAdapter
 from install_fuel_master import InstallFuelMaster
 from deploy_env import CloudDeploy
+from deploy_cache import DeployCache
 from execution_environment import ExecutionEnvironment

 from common import (
@@ -61,7 +62,8 @@ class AutoDeploy(object):
     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
-                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
+                 no_plugins, deploy_cache_dir, deploy_timeout,
+                 no_deploy_environment, deploy_log):
         self.no_fuel = no_fuel
         self.fuel_only = fuel_only
         self.no_health_check = no_health_check
@@ -75,6 +77,7 @@ class AutoDeploy(object):
         self.fuel_plugins_dir = fuel_plugins_dir
         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
         self.no_plugins = no_plugins
+        self.deploy_cache_dir = deploy_cache_dir
         self.deploy_timeout = deploy_timeout
         self.no_deploy_environment = no_deploy_environment
         self.deploy_log = deploy_log
@@ -116,7 +119,7 @@ class AutoDeploy(object):
                                   self.fuel_username, self.fuel_password,
                                   self.dea_file, self.fuel_plugins_conf_dir,
                                   WORK_DIR, self.no_health_check,
-                                  self.deploy_timeout,
+                                  self.deploy_cache_dir, self.deploy_timeout,
                                   self.no_deploy_environment, self.deploy_log)
             with old_dep.ssh:
                 old_dep.check_previous_installation()
@@ -128,6 +131,7 @@ class AutoDeploy(object):
                                  self.fuel_conf['ip'], self.fuel_username,
                                  self.fuel_password, self.fuel_node_id,
                                  self.iso_file, WORK_DIR,
+                                 self.deploy_cache_dir,
                                  self.fuel_plugins_dir, self.no_plugins)
         fuel.install()

@@ -136,6 +140,7 @@ class AutoDeploy(object):
         tmp_new_dir = '%s/newiso' % self.tmp_dir
         try:
             self.copy(tmp_orig_dir, tmp_new_dir)
+            self.deploy_cache_fingerprints(tmp_new_dir)
             self.patch(tmp_new_dir, new_iso)
         except Exception as e:
             exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
@@ -156,6 +161,12 @@ class AutoDeploy(object):
         delete(tmp_orig_dir)
         exec_cmd('chmod -R 755 %s' % tmp_new_dir)

+    def deploy_cache_fingerprints(self, tmp_new_dir):
+        if self.deploy_cache_dir:
+            log('Deploy cache: Collecting fingerprints...')
+            deploy_cache = DeployCache(self.deploy_cache_dir)
+            deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file)
+
     def patch(self, tmp_new_dir, new_iso):
         log('Patching...')
         patch_dir = '%s/%s' % (CWD, PATCH_DIR)
@@ -218,7 +229,8 @@ class AutoDeploy(object):
         dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
                           self.fuel_username, self.fuel_password,
                           self.dea_file, self.fuel_plugins_conf_dir,
-                          WORK_DIR, self.no_health_check, self.deploy_timeout,
+                          WORK_DIR, self.no_health_check,
+                          self.deploy_cache_dir, self.deploy_timeout,
                           self.no_deploy_environment, self.deploy_log)
         return dep.deploy()

@@ -343,6 +355,8 @@ def parse_arguments():
                         help='Fuel Plugins Configuration directory')
     parser.add_argument('-np', dest='no_plugins', action='store_true',
                         default=False, help='Do not install Fuel Plugins')
+    parser.add_argument('-dc', dest='deploy_cache_dir', action='store',
+                        help='Deploy Cache Directory')
     parser.add_argument('-dt', dest='deploy_timeout', action='store',
                         default=240, help='Deployment timeout (in minutes) '
                         '[default: 240]')
@@ -376,6 +390,10 @@ def parse_arguments():
         for bridge in args.pxe_bridge:
             check_bridge(bridge, args.dha_file)

+    if args.deploy_cache_dir:
+        log('Using deploy cache directory: %s' % args.deploy_cache_dir)
+        create_dir_if_not_exists(args.deploy_cache_dir)
+

     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
               'no_health_check': args.no_health_check,
@@ -386,6 +404,7 @@ def parse_arguments():
               'fuel_plugins_dir': args.fuel_plugins_dir,
               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
               'no_plugins': args.no_plugins,
+              'deploy_cache_dir': args.deploy_cache_dir,
               'deploy_timeout': args.deploy_timeout,
               'no_deploy_environment': args.no_deploy_environment,
               'deploy_log': args.deploy_log}
diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py
new file mode 100644
index 0000000..7df43c6
--- /dev/null
+++ b/deploy/deploy_cache.py
@@ -0,0 +1,312 @@
+###############################################################################
+# Copyright (c) 2016 Enea AB and others.
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+import glob
+import hashlib
+import io
+import json
+import os
+import shutil
+import yaml
+
+from common import (
+    exec_cmd,
+    log,
+)
+
+###############################################################################
+# Deploy Cache Flow Overview
+###############################################################################
+# 1. do_fingerprints
+#    Can be called as soon as a Fuel Master ISO chroot is available.
+#    This will gather all required information for uniquely identifying the
+#    objects in cache (bootstraps, targetimages).
+# 2. inject_cache
+#    Can be called as soon as we have a steady SSH connection to the Fuel
+#    Master node. It will inject cached artifacts over SSH, for later install.
+# 3. (external, async) install cached bootstrap instead of building a new one
+#    /sbin/bootstrap_admin_node.sh will check for cached bootstrap images
+#    (with id_rsa, id_rsa.pub attached) and will install those via
+#    $ fuel-bootstrap import opfnv-bootstraps-cache.tar
+# 4. install_targetimages_for_env
+#    Should be called before cloud deploy is started, to install env-generic
+#    'env_X_...' cached images for the current environment ID.
+#    Static method, to be used on the remote Fuel Master node; does not require
+#    access to the deploy cache, it only moves around some local files.
+# 5. extract_targetimages_from_env
+#    Should be called at env deploy finish, to prepare artifacts for caching.
+#    Static method, same observations as above apply.
+# 6. collect_artifacts
+#    Call last, to collect all artifacts.
+###############################################################################
+
+###############################################################################
+# Deploy cache artifacts:
+# - id_rsa
+# - bootstrap image (Ubuntu)
+# - environment target image (Ubuntu)
+###############################################################################
+# Cache fingerprint covers:
+# - bootstrap:
+#   - local mirror contents
+#   - package list (and everything else in fuel_bootstrap_cli.yaml)
+# - target image:
+#   - local mirror contents
+#   - package list (determined from DEA)
+###############################################################################
+# WARN: Cache fingerprint does NOT yet cover:
+# - image_data (always assume the default /boot, /);
+# - output_dir (always assume the default /var/www/nailgun/targetimages;
+# - codename (always assume the default, currently 'trusty');
+# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty
+# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub
+# - Auxiliary repo  .../mitaka-9.0/ubuntu/auxiliary
+# If the above change without triggering a cache miss, try clearing the cache.
+###############################################################################
+# WARN: Bootstrap caching implies RSA keypair to be reused!
+###############################################################################
+
+# Local mirrros will be used on Fuel Master for both bootstrap and target image
+# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`:
+# - MOS        .../mitaka-9.0/ubuntu/x86_64
+# - Ubuntu     .../mirrors/ubuntu/
+# All these reside on Fuel Master at local path:
+NAILGUN_PATH = '/var/www/nailgun/'
+
+# Artifact names (corresponding to nailgun subdirs)
+MIRRORS = 'mirrors'
+BOOTSTRAPS = 'bootstraps'
+TARGETIMAGES = 'targetimages'
+
+# Info for collecting RSA keypair
+RSA_KEYPAIR_PATH = '/root/.ssh'
+RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub']
+
+# Relative path for collecting the active bootstrap image(s) after env deploy
+NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS
+
+# Relative path for collecting target image(s) for deployed enviroment
+NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES
+
+# OPNFV Fuel bootstrap settings file that will be injected at deploy
+ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml'
+
+# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected
+REMOTE_CACHE_PATH = '/var/cache/opnfv'
+
+# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh
+BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar'
+
+# Env-ID indep prefix
+ENVX = 'env_X_'
+
+class DeployCache(object):
+    """OPNFV Deploy Cache - managed storage for cacheable artifacts"""
+
+    def __init__(self, cache_dir,
+                 fingerprints_yaml='deploy_cache_fingerprints.yaml'):
+        self.cache_dir = cache_dir
+        self.fingerprints_yaml = fingerprints_yaml
+        self.fingerprints = {BOOTSTRAPS: None,
+                             MIRRORS: None,
+                             TARGETIMAGES: None}
+
+    def __load_fingerprints(self):
+        """Load deploy cache yaml config holding fingerprints"""
+        if os.path.isfile(self.fingerprints_yaml):
+            cache_fingerprints = open(self.fingerprints_yaml).read()
+            self.fingerprints = yaml.load(cache_fingerprints)
+
+    def __save_fingerprints(self):
+        """Update deploy cache yaml config holding fingerprints"""
+        with open(self.fingerprints_yaml, 'w') as outfile:
+            outfile.write(yaml.safe_dump(self.fingerprints,
+                          default_flow_style=False))
+
+    def __fingerprint_mirrors(self, chroot_path):
+        """Collect repo mirror fingerprints"""
+        deb_packages = list()
+        # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins)
+        for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']:
+            for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)):
+                for fdeb in files:
+                    if fdeb.endswith(".deb"):
+                        deb_packages.append(fdeb)
+        sorted_debs = json.dumps(deb_packages, sort_keys=True)
+        self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest()
+
+    def __fingerprint_bootstrap(self, chroot_path):
+        """Collect bootstrap image metadata fingerprints"""
+        # FIXME(armband): include 'extra_dirs' contents
+        cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:])
+        bootstrap_cli_yaml = open(cli_yaml_path).read()
+        bootstrap_data = yaml.load(bootstrap_cli_yaml)
+        sorted_data = json.dumps(bootstrap_data, sort_keys=True)
+        self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest()
+
+    def __fingerprint_target(self, dea_file):
+        """Collect target image metadata fingerprints"""
+        # FIXME(armband): include 'image_data', 'codename', 'output'
+        with io.open(dea_file) as stream:
+            dea = yaml.load(stream)
+            editable = dea['settings']['editable']
+            target_data = {'packages': editable['provision']['packages'],
+                           'repos': editable['repo_setup']['repos']}
+            s_data = json.dumps(target_data, sort_keys=True)
+            self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest()
+
+    def do_fingerprints(self, chroot_path, dea_file):
+        """Collect SHA1 fingerprints based on chroot contents, DEA settings"""
+        try:
+            self.__load_fingerprints()
+            self.__fingerprint_mirrors(chroot_path)
+            self.__fingerprint_bootstrap(chroot_path)
+            self.__fingerprint_target(dea_file)
+            self.__save_fingerprints()
+        except Exception as ex:
+            log('Failed to get cache fingerprint: %s' % str(ex))
+
+    def __lookup_cache(self, sha):
+        """Search for object in cache based on SHA fingerprint"""
+        cache_sha_dir = os.path.join(self.cache_dir, sha)
+        if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir):
+            return None
+        return cache_sha_dir
+
+    def __inject_cache_dir(self, ssh, sha, artifact):
+        """Stage cached object (dir) in Fuel Master OPNFV local cache"""
+        local_path = self.__lookup_cache(sha)
+        if local_path:
+            remote_path = os.path.join(REMOTE_CACHE_PATH, artifact)
+            with ssh:
+                ssh.exec_cmd('mkdir -p %s' % remote_path)
+                for cachedfile in glob.glob('%s/*' % local_path):
+                    ssh.scp_put(cachedfile, remote_path)
+        return local_path
+
+    def __mix_fingerprints(self, f1, f2):
+        """Compute composite fingerprint"""
+        if self.fingerprints[f1] is None or self.fingerprints[f2] is None:
+            return None
+        return hashlib.sha1('%s%s' %
+            (self.fingerprints[f1], self.fingerprints[f2])).hexdigest()
+
+    def inject_cache(self, ssh):
+        """Lookup artifacts in cache and inject them over SSH/SCP into Fuel"""
+        try:
+            self.__load_fingerprints()
+            for artifact in [BOOTSTRAPS, TARGETIMAGES]:
+                sha = self.__mix_fingerprints(MIRRORS, artifact)
+                if sha is None:
+                    log('Missing fingerprint for: %s' % artifact)
+                    continue
+                if not self.__inject_cache_dir(ssh, sha, artifact):
+                    log('SHA1 not in cache: %s (%s)' % (str(sha), artifact))
+                else:
+                    log('SHA1 injected: %s (%s)' % (str(sha), artifact))
+        except Exception as ex:
+            log('Failed to inject cached artifacts into Fuel: %s' % str(ex))
+
+    def __extract_bootstraps(self, ssh, cache_sha_dir):
+        """Collect bootstrap artifacts from Fuel over SSH/SCP"""
+        remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE)
+        local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE)
+        with ssh:
+            for k in RSA_KEYPAIR_FILES:
+                ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k),
+                    local=os.path.join(cache_sha_dir, k))
+            ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' %
+                (REMOTE_CACHE_PATH,
+                os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR),
+                remote_tar))
+            ssh.scp_get(remote_tar, local=local_tar)
+            ssh.exec_cmd('rm -f %s' % remote_tar)
+
+    def __extract_targetimages(self, ssh, cache_sha_dir):
+        """Collect target image artifacts from Fuel over SSH/SCP"""
+        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
+        with ssh:
+            ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir)
+
+    def collect_artifacts(self, ssh):
+        """Collect artifacts from Fuel over SSH/SCP and add them to cache"""
+        try:
+            self.__load_fingerprints()
+            for artifact, func in {
+                    BOOTSTRAPS: self.__extract_bootstraps,
+                    TARGETIMAGES: self.__extract_targetimages
+                }.iteritems():
+                sha = self.__mix_fingerprints(MIRRORS, artifact)
+                if sha is None:
+                    log('WARN: Skip caching, NO fingerprint: %s' % artifact)
+                    continue
+                local_path = self.__lookup_cache(sha)
+                if local_path:
+                    log('SHA1 already in cache: %s (%s)' % (str(sha), artifact))
+                else:
+                    log('New cache SHA1: %s (%s)' % (str(sha), artifact))
+                    cache_sha_dir = os.path.join(self.cache_dir, sha)
+                    exec_cmd('mkdir -p %s' % cache_sha_dir)
+                    func(ssh, cache_sha_dir)
+        except Exception as ex:
+            log('Failed to extract artifacts from Fuel: %s' % str(ex))
+
+    @staticmethod
+    def extract_targetimages_from_env(env_id):
+        """Prepare targetimages from env ID for storage in deploy cache
+
+        NOTE: This method should be executed locally ON the Fuel Master node.
+        WARN: This method overwrites targetimages cache on Fuel Master node.
+        """
+        env_n = 'env_%s_' % str(env_id)
+        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
+        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
+        try:
+            exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path))
+            for root, _, files in os.walk(ti_path):
+                for tif in files:
+                    if tif.startswith(env_n):
+                        src = os.path.join(root, tif)
+                        dest = os.path.join(cti_path, tif.replace(env_n, ENVX))
+                        if tif.endswith('.yaml'):
+                            shutil.copy(src, dest)
+                            exec_cmd('sed -i "s|%s|%s|g" %s' %
+                                     (env_n, ENVX, dest))
+                        else:
+                            os.link(src, dest)
+        except Exception as ex:
+            log('Failed to extract targetimages artifacts from env %s: %s' %
+                (str(env_id), str(ex)))
+
+    @staticmethod
+    def install_targetimages_for_env(env_id):
+        """Install targetimages artifacts for a specific env ID
+
+        NOTE: This method should be executed locally ON the Fuel Master node.
+        """
+        env_n = 'env_%s_' % str(env_id)
+        cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES)
+        ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR)
+        if not os.path.isdir(cti_path):
+            log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path))
+        else:
+            try:
+                for root, _, files in os.walk(cti_path):
+                    for tif in files:
+                        src = os.path.join(root, tif)
+                        dest = os.path.join(ti_path, tif.replace(ENVX, env_n))
+                        if tif.endswith('.yaml'):
+                            shutil.copy(src, dest)
+                            exec_cmd('sed -i "s|%s|%s|g" %s' %
+                                     (ENVX, env_n, dest))
+                        else:
+                            os.link(src, dest)
+            except Exception as ex:
+                log('Failed to install targetimages for env %s: %s' %
+                    (str(env_id), str(ex)))
diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
index d374cce..445070a 100644
--- a/deploy/deploy_env.py
+++ b/deploy/deploy_env.py
@@ -15,6 +15,7 @@ import glob
 import time
 import shutil

+from deploy_cache import DeployCache
 from ssh_client import SSHClient

 from common import (
@@ -35,7 +36,8 @@ class CloudDeploy(object):

     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
                  dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
-                 deploy_timeout, no_deploy_environment, deploy_log):
+                 deploy_cache_dir, deploy_timeout,
+                 no_deploy_environment, deploy_log):
         self.dea = dea
         self.dha = dha
         self.fuel_ip = fuel_ip
@@ -49,6 +51,8 @@ class CloudDeploy(object):
         self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
         self.work_dir = work_dir
         self.no_health_check = no_health_check
+        self.deploy_cache = ( DeployCache(deploy_cache_dir)
+                              if deploy_cache_dir else None )
         self.deploy_timeout = deploy_timeout
         self.no_deploy_environment = no_deploy_environment
         self.deploy_log = deploy_log
@@ -82,9 +86,14 @@ class CloudDeploy(object):
                 self.work_dir, os.path.basename(self.dea_file)))
             s.scp_put('%s/common.py' % self.file_dir, self.work_dir)
             s.scp_put('%s/dea.py' % self.file_dir, self.work_dir)
+            s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir)
             for f in glob.glob('%s/cloud/*' % self.file_dir):
                 s.scp_put(f, self.work_dir)

+    def deploy_cache_collect_artifacts(self):
+        if self.deploy_cache:
+            self.deploy_cache.collect_artifacts(self.ssh)
+
     def power_off_nodes(self):
         for node_id in self.node_ids:
             self.dha.node_power_off(node_id)
@@ -281,4 +290,6 @@ class CloudDeploy(object):

         self.get_put_deploy_log()

+        self.deploy_cache_collect_artifacts()
+
         return rc
diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
index b731c6b..83d31fb 100644
--- a/deploy/install_fuel_master.py
+++ b/deploy/install_fuel_master.py
@@ -10,6 +10,7 @@
 import time
 import os
 import glob
+from deploy_cache import DeployCache
 from ssh_client import SSHClient
 from dha_adapters.libvirt_adapter import LibvirtAdapter

@@ -32,7 +33,7 @@ class InstallFuelMaster(object):

     def __init__(self, dea_file, dha_file, fuel_ip, fuel_username,
                  fuel_password, fuel_node_id, iso_file, work_dir,
-                 fuel_plugins_dir, no_plugins):
+                 deploy_cache_dir, fuel_plugins_dir, no_plugins):
         self.dea_file = dea_file
         self.dha = LibvirtAdapter(dha_file)
         self.fuel_ip = fuel_ip
@@ -42,6 +43,8 @@ class InstallFuelMaster(object):
         self.iso_file = iso_file
         self.iso_dir = os.path.dirname(self.iso_file)
         self.work_dir = work_dir
+        self.deploy_cache = ( DeployCache(deploy_cache_dir)
+                              if deploy_cache_dir else None )
         self.fuel_plugins_dir = fuel_plugins_dir
         self.no_plugins = no_plugins
         self.file_dir = os.path.dirname(os.path.realpath(__file__))
@@ -83,6 +86,10 @@ class InstallFuelMaster(object):
         log('Wait until Fuel menu is up')
         fuel_menu_pid = self.wait_until_fuel_menu_up()

+        if self.deploy_cache:
+            log('Deploy cache: Injecting bootstraps and targetimages')
+            self.deploy_cache.inject_cache(self.ssh)
+
         log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings')
         self.inject_own_astute_and_bootstrap_yaml()