summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexandru Avadanii <Alexandru.Avadanii@enea.com>2018-11-21 16:57:10 +0100
committerAlexandru Avadanii <Alexandru.Avadanii@enea.com>2018-11-21 18:14:15 +0100
commitf03a33bc58135acc7e4420efaaf34c26a32d6142 (patch)
tree8c302374639934a4d842bc79f9b8ebf4e6d957db
parent3fef67294b7e2b880217e2e1e0cafee6e4d6e9bc (diff)
[docker] build: Allow cache invalidation
While at it, fix emoji issues with latest virtualenv [1]. JIRA: FUEL-398 [1] https://github.com/pypa/pipenv/issues/3223 Change-Id: Ice5937222bf75c1ddadc6b9f1994635bc10faf57 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
-rwxr-xr-xci/build.sh11
-rw-r--r--mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch59
-rw-r--r--mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch (renamed from mcp/patches/docker/0001-OPNFV-tag-convention-alignment.patch)16
-rw-r--r--mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch (renamed from mcp/patches/docker/0002-OPNFV-package-installation-Ubuntu-user.patch)25
-rw-r--r--mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch (renamed from mcp/patches/docker/0003-reclass-Set-ignore_overwritten_missing_references.patch)0
5 files changed, 87 insertions, 24 deletions
diff --git a/ci/build.sh b/ci/build.sh
index 67b6d46d8..108ce139a 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -32,12 +32,14 @@ DEPLOY_DIR=$(cd "${MCP_REPO_ROOT_PATH}/mcp/scripts"; pwd)
DOCKER_DIR=$(cd "${MCP_REPO_ROOT_PATH}/docker"; pwd)
DOCKER_TAG=${1:-latest}
DOCKER_PUSH=${2---push} # pass an empty second arg to disable push
+CACHE_INVALIDATE=${CACHE_INVALIDATE:-0}
source "${DEPLOY_DIR}/globals.sh"
source "${DEPLOY_DIR}/lib.sh"
source "${DEPLOY_DIR}/lib_jump_common.sh"
[ ! "${TERM:-unknown}" = 'unknown' ] || export TERM=vt220
+[ "${CACHE_INVALIDATE}" = 0 ] || CACHE_INVALIDATE=$(date +%s)
#
# END of variables to customize
@@ -69,11 +71,10 @@ docker_install
popd > /dev/null
pushd "${DOCKER_DIR}" > /dev/null
-python -m pipenv --two
-env VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install
-env VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install invoke
+env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv --two install
+env PIPENV_HIDE_EMOJIS=1 VIRTUALENV_ALWAYS_COPY=1 python -m pipenv install invoke
# shellcheck disable=SC2086
-python -m pipenv run \
+env PIPENV_HIDE_EMOJIS=1 python -m pipenv run \
invoke build saltmaster-reclass \
--require 'salt salt-formulas opnfv reclass tini-saltmaster' \
--dist=ubuntu \
@@ -81,6 +82,8 @@ python -m pipenv run \
--formula-rev=nightly \
--opnfv-tag="${DOCKER_TAG}" \
--salt='stable 2017.7' \
+ --build-arg-extra " \
+ CACHE_INVALIDATE=\"${CACHE_INVALIDATE}\"" \
${DOCKER_PUSH}
popd > /dev/null
diff --git a/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch b/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch
new file mode 100644
index 000000000..cf1c57d31
--- /dev/null
+++ b/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch
@@ -0,0 +1,59 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 9 Sep 2018 23:18:39 +0200
+Subject: [PATCH] tasks.py: Allow passing extra --build-arg
+
+E.g. A, B with values "a", respectively "b c":
+$ invoke build saltmaster-reclass --build-arg-extra='A="a" B="b c"'
+will be passed down as:
+--build-arg A="a" --build-arg B="b c"
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ tasks.py | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/tasks.py b/tasks.py
+index d6bf3bc..59e217d 100644
+--- a/tasks.py
++++ b/tasks.py
+@@ -3,6 +3,7 @@
+
+
+ from invoke import Collection, task
++from shlex import split
+ from string import Template
+ import re
+ import ast
+@@ -25,7 +26,7 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs):
+ dry=dry, push=push, dry_targets=dry_targets, filter=filter, **kwargs)
+
+ @task
+-def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, **kwargs):
++def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs):
+
+ kwargs['dist'] = dist
+ kwargs['dist_rel'] = dist_rel
+@@ -35,6 +36,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
+ kwargs['require'] = require
+ kwargs['salt'] = salt
+ kwargs['target'] = target
++ kwargs['build_arg_extra'] = ' --build-arg '.join([''] + split(build_arg_extra.replace('"', '"\\"')))
+ # command formating + update
+ fmt = {'tag': ''}
+ fmt.update(ctx.dockermake)
+@@ -60,6 +62,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
+ \t--requires ${requires} \
+ \t--build-arg SALT_VERSION="${salt}" \
+ \t--build-arg SALT_FORMULA_VERSION="${formula_rev}" \
++ \t${build_arg_extra} \
+ \t${push} ${options} \
+ ${fin}""").safe_substitute(fmt)
+ ctx.run(cmd.replace(' ', ''))
diff --git a/mcp/patches/docker/0001-OPNFV-tag-convention-alignment.patch b/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch
index aa0b01c39..6ceb4873d 100644
--- a/mcp/patches/docker/0001-OPNFV-tag-convention-alignment.patch
+++ b/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch
@@ -37,10 +37,10 @@ index 0bd793e..291f4c7 100644
# options: '--bust-cache reclass'
# options: '--registry-user epcim'
diff --git a/tasks.py b/tasks.py
-index d6bf3bc..092c6e7 100644
+index 59e217d..8afbde9 100644
--- a/tasks.py
+++ b/tasks.py
-@@ -6,6 +6,7 @@ from invoke import Collection, task
+@@ -7,6 +7,7 @@ from shlex import split
from string import Template
import re
import ast
@@ -48,26 +48,26 @@ index d6bf3bc..092c6e7 100644
# TODOs:
# - WIP - use namespace to expose images/targets as inoke tasks # image = Collection('image')
-@@ -25,8 +26,9 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs):
+@@ -26,8 +27,9 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs):
dry=dry, push=push, dry_targets=dry_targets, filter=filter, **kwargs)
@task
--def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, **kwargs):
-+def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, opnfv_tag='latest', push=False, dry=False, dry_targets=False, **kwargs):
+-def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs):
++def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, opnfv_tag='latest', push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs):
+ kwargs['arch'] = 'arm64' if platform.machine() == 'aarch64' else 'amd64'
kwargs['dist'] = dist
kwargs['dist_rel'] = dist_rel
kwargs['dry'] = True if dry_targets or dry else False
-@@ -35,6 +37,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
- kwargs['require'] = require
+@@ -37,6 +39,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
kwargs['salt'] = salt
kwargs['target'] = target
+ kwargs['build_arg_extra'] = ' --build-arg '.join([''] + split(build_arg_extra.replace('"', '"\\"')))
+ kwargs['opnfv_tag'] = opnfv_tag
# command formating + update
fmt = {'tag': ''}
fmt.update(ctx.dockermake)
-@@ -56,7 +59,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
+@@ -58,7 +61,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None,
# execute
cmd = Template("""
${dry}docker-make -f DockerMake.${dist}.yml -u ${repository}: --name ${target} \
diff --git a/mcp/patches/docker/0002-OPNFV-package-installation-Ubuntu-user.patch b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
index a96c24b1f..d283d06b6 100644
--- a/mcp/patches/docker/0002-OPNFV-package-installation-Ubuntu-user.patch
+++ b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
@@ -21,14 +21,22 @@ Subject: [PATCH] OPNFV package installation, Ubuntu user
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---
- DockerMake.yml | 38 ++++++++++++++++++++++++++++++++++++--
- 1 file changed, 36 insertions(+), 2 deletions(-)
+ DockerMake.yml | 37 ++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 36 insertions(+), 1 deletion(-)
diff --git a/DockerMake.yml b/DockerMake.yml
-index 2c75586..13813a2 100644
+index 2c75586..eb06cbd 100644
--- a/DockerMake.yml
+++ b/DockerMake.yml
-@@ -94,7 +94,7 @@ salt-formulas:
+@@ -29,6 +29,7 @@ common-cleanup:
+
+ common:
+ build: |
++ ARG CACHE_INVALIDATE="0"
+ RUN echo "Layer with common packages" \
+ && eval ${LAYER_PKGUPDT} \
+ && apt-get upgrade -qy \
+@@ -94,7 +95,7 @@ salt-formulas:
build: |
ARG SALT_FORMULA_VERSION="master"
ENV SALT_FORMULA_VERSION $SALT_FORMULA_VERSION
@@ -37,14 +45,7 @@ index 2c75586..13813a2 100644
ENV SALT_FORMULA_SOURCES $SALT_FORMULA_SOURCES
ARG SALT_FORMULAS_BASE="/srv/salt/formula"
ENV SALT_FORMULAS_BASE $SALT_FORMULAS_BASE
-@@ -102,12 +102,46 @@ salt-formulas:
- ENV SALT_ENV_PATH_ $SALT_ENV_PATH_
- ARG RECLASS_BASE="/srv/salt/reclass"
- ENV RECLASS_BASE $RECLASS_BASE
-- RUN echo "Layer python/salt module prerequisites, formulas" \
-+ RUN echo "Layer python/salt module prerequisites, formulas (09 Nov 2018)" \
- && mkdir -p /srv/salt \
- && curl -sSqL https://raw.githubusercontent.com/salt-formulas/salt-formulas-scripts/master/formula-fetch.sh -o /srv/salt/formula-fetch.sh \
+@@ -108,6 +109,40 @@ salt-formulas:
&& bash -c 'source /srv/salt/formula-fetch.sh && setupPyEnv && fetchAll' \
&& eval ${LAYER_CLEANUP}
diff --git a/mcp/patches/docker/0003-reclass-Set-ignore_overwritten_missing_references.patch b/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch
index 4437ecd91..4437ecd91 100644
--- a/mcp/patches/docker/0003-reclass-Set-ignore_overwritten_missing_references.patch
+++ b/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch