Merge "[docker] build: Allow cache invalidation"

5 files changed, 87 insertions, 24 deletions

diff --git a/ci/build.sh b/ci/build.sh
index 67b6d46d8..108ce139a 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -32,12 +32,14 @@ DEPLOY_DIR=$(cd "${MCP_REPO_ROOT_PATH}/mcp/scripts"; pwd)
 DOCKER_DIR=$(cd "${MCP_REPO_ROOT_PATH}/docker"; pwd)
 DOCKER_TAG=${1:-latest}
 DOCKER_PUSH=${2---push}  # pass an empty second arg to disable push
+CACHE_INVALIDATE=${CACHE_INVALIDATE:-0}
 
 source "${DEPLOY_DIR}/globals.sh"
 source "${DEPLOY_DIR}/lib.sh"
 source "${DEPLOY_DIR}/lib_jump_common.sh"
 
 [ ! "${TERM:-unknown}" = 'unknown' ] || export TERM=vt220
+[ "${CACHE_INVALIDATE}" = 0 ] || CACHE_INVALIDATE=$(date +%s)
 
 #
 # END of variables to customize
@@ -69,11 +71,10 @@ docker_install
 popd > /dev/null
 pushd "${DOCKER_DIR}" > /dev/null
 
-python -m pipenv --two
-env VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install
-env VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install invoke
+env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv --two install
+env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install invoke
 # shellcheck disable=SC2086
-python -m pipenv run \
+env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \
   invoke build saltmaster-reclass \
     --require 'salt salt-formulas opnfv reclass tini-saltmaster' \
     --dist=ubuntu \
@@ -81,6 +82,8 @@ python -m pipenv run \
     --formula-rev=nightly \
     --opnfv-tag="${DOCKER_TAG}" \
     --salt='stable 2017.7' \
+    --build-arg-extra " \
+        CACHE_INVALIDATE=\"${CACHE_INVALIDATE}\"" \
     ${DOCKER_PUSH}
 
 popd > /dev/null
diff --git a/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch b/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch
new file mode 100644
index 000000000..cf1c57d31
--- /dev/null
+++ b/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch
@@ -0,0 +1,59 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 9 Sep 2018 23:18:39 +0200
+Subject: [PATCH] tasks.py: Allow passing extra --build-arg
+
+E.g. A, B with values "a", respectively "b c":
+$ invoke build saltmaster-reclass --build-arg-extra='A="a" B="b c"'
+will be passed down as:
+--build-arg A="a" --build-arg B="b c"
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ tasks.py | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/tasks.py b/tasks.py
+index d6bf3bc..59e217d 100644
+--- a/tasks.py
++++ b/tasks.py
+@@ -3,6 +3,7 @@
+
+
+ from invoke import Collection, task
++from shlex import split
+ from string import Template
+ import re
+ import ast
+@@ -25,7 +26,7 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs):
+                      dry=dry, push=push, dry_targets=dry_targets, filter=filter, **kwargs)
+
+ @task
+-def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, **kwargs):
++def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs):
+
+     kwargs['dist'] = dist
+     kwargs['dist_rel'] = dist_rel
+@@ -35,6 +36,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
+     kwargs['require'] = require
+     kwargs['salt'] = salt
+     kwargs['target'] = target
++    kwargs['build_arg_extra'] = ' --build-arg '.join([''] + split(build_arg_extra.replace('"', '"\\"')))
+     # command formating + update
+     fmt = {'tag': ''}
+     fmt.update(ctx.dockermake)
+@@ -60,6 +62,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
+             \t--requires ${requires} \
+             \t--build-arg SALT_VERSION="${salt}" \
+             \t--build-arg SALT_FORMULA_VERSION="${formula_rev}" \
++            \t${build_arg_extra} \
+             \t${push} ${options} \
+             ${fin}""").safe_substitute(fmt)
+     ctx.run(cmd.replace('  ', ''))
diff --git a/mcp/patches/docker/0001-OPNFV-tag-convention-alignment.patch b/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch
index aa0b01c39..6ceb4873d 100644
--- a/mcp/patches/docker/0001-OPNFV-tag-convention-alignment.patch
+++ b/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch
@@ -37,10 +37,10 @@ index 0bd793e..291f4c7 100644
    # options: '--bust-cache reclass'
    # options: '--registry-user epcim'
 diff --git a/tasks.py b/tasks.py
-index d6bf3bc..092c6e7 100644
+index 59e217d..8afbde9 100644
 --- a/tasks.py
 +++ b/tasks.py
-@@ -6,6 +6,7 @@ from invoke import Collection, task
+@@ -7,6 +7,7 @@ from shlex import split
  from string import Template
  import re
  import ast
@@ -48,26 +48,26 @@ index d6bf3bc..092c6e7 100644
 
  # TODOs:
  # - WIP - use namespace to expose images/targets as inoke tasks # image = Collection('image')
-@@ -25,8 +26,9 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs):
+@@ -26,8 +27,9 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs):
                       dry=dry, push=push, dry_targets=dry_targets, filter=filter, **kwargs)
 
  @task
--def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, **kwargs):
-+def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, opnfv_tag='latest', push=False, dry=False, dry_targets=False, **kwargs):
+-def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs):
++def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, opnfv_tag='latest', push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs):
 
 +    kwargs['arch'] = 'arm64' if platform.machine() == 'aarch64' else 'amd64'
      kwargs['dist'] = dist
      kwargs['dist_rel'] = dist_rel
      kwargs['dry'] = True if dry_targets or dry else False
-@@ -35,6 +37,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
-     kwargs['require'] = require
+@@ -37,6 +39,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
      kwargs['salt'] = salt
      kwargs['target'] = target
+     kwargs['build_arg_extra'] = ' --build-arg '.join([''] + split(build_arg_extra.replace('"', '"\\"')))
 +    kwargs['opnfv_tag'] = opnfv_tag
      # command formating + update
      fmt = {'tag': ''}
      fmt.update(ctx.dockermake)
-@@ -56,7 +59,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
+@@ -58,7 +61,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
      # execute
      cmd = Template("""
              ${dry}docker-make -f DockerMake.${dist}.yml -u ${repository}: --name ${target} \
diff --git a/mcp/patches/docker/0002-OPNFV-package-installation-Ubuntu-user.patch b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
index a96c24b1f..d283d06b6 100644
--- a/mcp/patches/docker/0002-OPNFV-package-installation-Ubuntu-user.patch
+++ b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
@@ -21,14 +21,22 @@ Subject: [PATCH] OPNFV package installation, Ubuntu user
 
 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
 ---
- DockerMake.yml | 38 ++++++++++++++++++++++++++++++++++++--
- 1 file changed, 36 insertions(+), 2 deletions(-)
+ DockerMake.yml | 37 ++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 36 insertions(+), 1 deletion(-)
 
 diff --git a/DockerMake.yml b/DockerMake.yml
-index 2c75586..13813a2 100644
+index 2c75586..eb06cbd 100644
 --- a/DockerMake.yml
 +++ b/DockerMake.yml
-@@ -94,7 +94,7 @@ salt-formulas:
+@@ -29,6 +29,7 @@ common-cleanup:
+
+ common:
+   build: |
++    ARG CACHE_INVALIDATE="0"
+     RUN echo "Layer with common packages" \
+       && eval ${LAYER_PKGUPDT} \
+       && apt-get upgrade -qy \
+@@ -94,7 +95,7 @@ salt-formulas:
    build: |
      ARG SALT_FORMULA_VERSION="master"
      ENV SALT_FORMULA_VERSION $SALT_FORMULA_VERSION
@@ -37,14 +45,7 @@ index 2c75586..13813a2 100644
      ENV SALT_FORMULA_SOURCES $SALT_FORMULA_SOURCES
      ARG SALT_FORMULAS_BASE="/srv/salt/formula"
      ENV SALT_FORMULAS_BASE $SALT_FORMULAS_BASE
-@@ -102,12 +102,46 @@ salt-formulas:
-     ENV SALT_ENV_PATH_ $SALT_ENV_PATH_
-     ARG RECLASS_BASE="/srv/salt/reclass"
-     ENV RECLASS_BASE $RECLASS_BASE
--    RUN echo "Layer python/salt module prerequisites, formulas" \
-+    RUN echo "Layer python/salt module prerequisites, formulas (09 Nov 2018)" \
-       && mkdir -p /srv/salt \
-       && curl -sSqL https://raw.githubusercontent.com/salt-formulas/salt-formulas-scripts/master/formula-fetch.sh -o /srv/salt/formula-fetch.sh \
+@@ -108,6 +109,40 @@ salt-formulas:
        && bash -c 'source /srv/salt/formula-fetch.sh && setupPyEnv && fetchAll' \
        && eval ${LAYER_CLEANUP}
 
diff --git a/mcp/patches/docker/0003-reclass-Set-ignore_overwritten_missing_references.patch b/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch
index 4437ecd91..4437ecd91 100644
--- a/mcp/patches/docker/0003-reclass-Set-ignore_overwritten_missing_references.patch
+++ b/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch