From cb52dd523c58a6567b53ea5b79169295e1471117 Mon Sep 17 00:00:00 2001
From: Cédric Ollivier <cedric.ollivier@orange.com>
Date: Tue, 10 Oct 2017 15:44:00 +0200
Subject: Publish Docker Slicing Presentation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It published the presentation done in the Functest Breakout during
last OPNFV Summit (2017 in Beijing) [1].

Minor changes are planned to update it in a second change.

Git now ignores pdf which may be generated via reveal.js.

[1] http://testresults.opnfv.org/functest/dockerslicing/

Change-Id: I2b6cf113c672320bf1bc34442bb267fd99012546
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
---
 .gitignore                                   |   1 +
 docs/com/pres/dockerslicing/dockerslicing.md | 170 +++++++++++++++++++++++++++
 docs/com/pres/dockerslicing/index.html       |  52 ++++++++
 3 files changed, 223 insertions(+)
 create mode 100644 docs/com/pres/dockerslicing/dockerslicing.md
 create mode 100644 docs/com/pres/dockerslicing/index.html

diff --git a/.gitignore b/.gitignore
index f6e406203..81ebca89c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ functest.egg-info
 *venv
 .tox
 *.out
+*.pdf
diff --git a/docs/com/pres/dockerslicing/dockerslicing.md b/docs/com/pres/dockerslicing/dockerslicing.md
new file mode 100644
index 000000000..55fb2557a
--- /dev/null
+++ b/docs/com/pres/dockerslicing/dockerslicing.md
@@ -0,0 +1,170 @@
+# Docker slicing
+
+[Cédric Ollivier](mailto:cedric.ollivier@orange.com)
+
+OPNVF Summit 2017 (Beijing)
+
+
+
+## Current issues
+
+
+### Dockerfile
+
+- it copies all the files hosted by the third-party projects (eg docs, .git...)
+- several requirements are downgraded/upgraded when building the container as they are managed one after the other
+- it could download packages from [PyPI](https://pypi.python.org/pypi) (e.g. [rally](https://pypi.python.org/pypi/rally/)...) instead of cloning git repository
+- build dependencies can't be removed to save space as it creates multiple layers (>70)
+
+
+### setup.py
+
+- no requirement is installed when calling *python setup.py install* as none of the next keys are set:
+    - install_requires
+    - tests_require
+    - dependency_links
+- shell scripts are not installed neither in $PATH nor in dist-packages
+
+
+### setup.py
+
+- master is considered as an incorrect version
+- empty package_data seems useless
+- zip_safe flag could be set as False as \__file__ would prevent functest from working in a zipfile
+- py_modules should not list cli_base as it was moved into functest package
+
+
+
+## Proposals
+
+
+### why not relying on [pbr](https://docs.openstack.org/developer/pbr/)?
+
+- pbr injects requirements into the install_requires, tests_require and/or dependency_links arguments to setup
+- it supports conditional dependencies which can be added to the requirements (e.g. subprocess32; python_version=='2.7')
+
+
+### setup.py
+
+```python
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup(
+    setup_requires=['pbr>=1.9', 'setuptools>=17.1'],
+    pbr=True,
+)
+```
+
+
+### setup.cfg
+
+```ini
+[metadata]
+name = functest
+version = 5
+home-page = https://wiki.opnfv.org/display/functest
+
+[files]
+packages = functest
+scripts =
+    docker/docker_remote_api/enable_remote_api.sh
+    docker/add_images.sh
+    docker/config_install_env.sh
+
+[entry_points]
+console_scripts =
+    functest = functest.cli.cli_base:cli
+```
+
+
+### Split requirements into (at least) 3 files
+
+- **requirements.txt** which could list all abstract (i.e. [not associated with any particular index](https://packaging.python.org/requirements/)) dependencies of the functest package
+- **test-requirements.txt** which could list all abstract dependencies required for testing the functest package
+- **thirdparty-requirements.txt** which could list all abstract and concrete dependencies required by the full functest docker container
+
+
+### tox.ini
+
+```ini
+[testenv]
+usedevelop = True
+deps =
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+  git+https://gerrit.opnfv.org/gerrit/releng#egg=opnfv&subdirectory=modules
+  git+https://gerrit.opnfv.org/gerrit/barometer#egg=baro_tests
+  git+https://gerrit.opnfv.org/gerrit/snaps#egg=snaps
+
+```
+
+
+### Dockerfile
+
+```
+RUN pip install \
+  git+https://gerrit.opnfv.org/gerrit/functest@$BRANCH#egg=functest \
+  git+https://gerrit.opnfv.org/gerrit/releng@$BRANCH#egg=opnfv\&subdirectory=modules \
+  git+https://gerrit.opnfv.org/gerrit/barometer@$BRANCH#egg=baro_tests \
+  git+https://gerrit.opnfv.org/gerrit/snaps@$BRANCH#egg=snaps
+```
+
+
+### Switch to Alpine
+
+```
+FROM alpine:3.5
+
+RUN apk --no-cache add --update \
+        python libffi libssl1.0 libjpeg-turbo py-pip && \
+    apk --no-cache add --virtual .build-deps --update \
+        python-dev build-base linux-headers libffi-dev \
+        openssl-dev libjpeg-turbo-dev git && \
+    pip install git+https://gerrit.opnfv.org/gerrit/releng#egg=opnfv\&subdirectory=modules \
+        git+https://gerrit.opnfv.org/gerrit/barometer#egg=baro_tests \
+        git+https://gerrit.opnfv.org/gerrit/snaps#egg=snaps \
+        git+https://gerrit.opnfv.org/gerrit/functest@$BRANCH#egg=functest && \
+    apk del .build-deps
+```
+
+
+### Status
+
+ - the change [**"leveraging on pbr"**](https://gerrit.opnfv.org/gerrit/#/c/35813/) can be merged
+ - docker containers have been published (any test is welcome):
+    - [ollivier/functest-pbr](https://hub.docker.com/r/ollivier/functest-pbr/)
+    - [ollivier/functest-alpine](https://hub.docker.com/r/ollivier/functest-alpine/)
+ - the Dockerfile switching to Alpine can be published too
+
+
+
+## Next steps
+
+
+### docker slicing
+
+the previous alpine example could be the base image and we could simply create other images from it (see [FROM](https://docs.docker.com/engine/reference/builder/#from)) by installing all third party python packages
+
+
+### PyPI
+
+functest could be published to the [Python Package Index](https://pypi.python.org/pypi) and then we would stop listing urls
+
+
+
+## on the road
+
+
+### We could be faced with issues
+
+- several test cases can fail if their requirements are incomplete in setup.py (we should help them to fix it)
+- all conditions for requirements may not be compatible (OpenStack requirements as a guideline?)
+- we should fix lots of hardcoded paths hidden by the previous design
+- some files could be outside the python package (e.g. functest ci scripts)
+
+
+
+## Thank You!
+
diff --git a/docs/com/pres/dockerslicing/index.html b/docs/com/pres/dockerslicing/index.html
new file mode 100644
index 000000000..1c65e4fa0
--- /dev/null
+++ b/docs/com/pres/dockerslicing/index.html
@@ -0,0 +1,52 @@
+<html>
+<head>
+<title>Docker slicing</title>
+<meta name="author" content="Cédric Ollivier">
+<meta name="viewport"
+    content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+<link rel="stylesheet" href="../reveal.js/css/reveal.css">
+<link rel="stylesheet" href="../reveal.js/css/theme/white.css">
+<link rel="stylesheet" href="../reveal.js/lib/css/zenburn.css">
+<script>
+var link = document.createElement( 'link' );
+link.rel = 'stylesheet';
+link.type = 'text/css';
+link.href = window.location.search.match( /print-pdf/gi ) ? '../reveal.js/css/print/pdf.css' : '../reveal.js/css/print/paper.css';
+document.getElementsByTagName( 'head' )[0].appendChild( link );
+</script>
+</head>
+<body>
+    <div class="reveal">
+        <div class="slides">
+            <section data-markdown="dockerslicing.md" data-separator="^\n\n\n"
+                data-separator-vertical="^\n\n" data-separator-notes="^Note:"></section>
+        </div>
+    </div>
+    <script src="../reveal.js/lib/js/head.min.js"></script>
+    <script src="../reveal.js/js/reveal.js"></script>
+    <script>
+        Reveal.initialize({
+            dependencies : [ {
+                src : '../reveal.js/plugin/markdown/marked.js',
+                condition : function() {
+                    return !!document.querySelector('[data-markdown]');
+                }
+            }, {
+                src : '../reveal.js/plugin/markdown/markdown.js',
+                condition : function() {
+                    return !!document.querySelector('[data-markdown]');
+                }
+            }, {
+                src: '../reveal.js/plugin/highlight/highlight.js',
+                async: true,
+                callback: function() {
+                    hljs.initHighlightingOnLoad();
+                }
+            }, {
+            src: '../reveal.js/plugin/notes/notes.js',
+        async: true
+            } ]
+        });
+    </script>
+</body>
+</html>
-- 
cgit 1.2.3-korg


From ed777b643b05fbb71851df0b883483e0bcaa4ac1 Mon Sep 17 00:00:00 2001
From: Cédric Ollivier <cedric.ollivier@orange.com>
Date: Wed, 11 Oct 2017 05:29:11 +0200
Subject: Allow building aarch64 images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It must be noted that Docker automated build cannot create these
images as the prerequisites are unmet to cross-compile [1].

It also publishes a travis config file [2].

[1] https://www.kernel.org/doc/html/v4.11/admin-guide/binfmt-misc.html
[2] https://travis-ci.org/collivier/functest/builds/286932633

Change-Id: I11fdef410cc40989132205a098410175f789fdef
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
---
 .travis.yml | 133 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 build.sh    |  23 +++++++----
 2 files changed, 149 insertions(+), 7 deletions(-)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000000000..af67f4786
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,133 @@
+sudo: required
+language: bash
+services: docker
+language: generic
+
+before_script:
+  - sudo apt-get -y install qemu-user-static
+  - sudo -E docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
+  - (cd .. && git clone https://github.com/estesp/manifest-tool)
+  - (cd ../manifest-tool && sudo -E make && sudo -E make install)
+
+jobs:
+  include:
+    - stage: build functest-core images
+      script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs=""
+        - aarch64_dirs="docker/core"
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/core"
+        - aarch64_dirs=""
+    - stage: publish functest-core manifests
+      script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${DOCKER_USERNAME}/functest-core:x86_64-latest \
+          --template ${DOCKER_USERNAME}/functest-core:aarch64-latest \
+          --target ${DOCKER_USERNAME}/functest-core:latest
+    - stage: build all functest images
+      script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs=""
+        - aarch64_dirs="docker/healthcheck"
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs=""
+        - aarch64_dirs="docker/smoke"
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs=""
+        - aarch64_dirs="docker/features"
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs=""
+        - aarch64_dirs="docker/components"
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/healthcheck"
+        - aarch64_dirs=""
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/smoke"
+        - aarch64_dirs=""
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/features"
+        - aarch64_dirs=""
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/components"
+        - aarch64_dirs=""
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/vnf"
+        - aarch64_dirs=""
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/restapi"
+        - aarch64_dirs=""
+    - script: sudo -E bash build.sh
+      env:
+        - repo="${DOCKER_USERNAME}"
+        - x86_64_dirs="docker/parser"
+        - aarch64_dirs=""
+    - stage: publish all manifests
+      script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${DOCKER_USERNAME}/functest-healthcheck:x86_64-latest \
+          --template ${DOCKER_USERNAME}/functest-healthcheck:aarch64-latest \
+          --target ${DOCKER_USERNAME}/functest-healthcheck:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${DOCKER_USERNAME}/functest-healthcheck:x86_64-latest \
+          --template ${DOCKER_USERNAME}/functest-healthcheck:aarch64-latest \
+          --target ${DOCKER_USERNAME}/functest-healthcheck:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${DOCKER_USERNAME}/functest-smoke:x86_64-latest \
+          --template ${DOCKER_USERNAME}/functest-smoke:aarch64-latest \
+          --target ${DOCKER_USERNAME}/functest-smoke:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${DOCKER_USERNAME}/functest-features:x86_64-latest \
+          --template ${DOCKER_USERNAME}/functest-features:aarch64-latest \
+          --target ${DOCKER_USERNAME}/functest-features:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64,linux/arm64 \
+          --template ${DOCKER_USERNAME}/functest-components:x86_64-latest \
+          --template ${DOCKER_USERNAME}/functest-components:aarch64-latest \
+          --target ${DOCKER_USERNAME}/functest-components:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64 \
+          --template ${DOCKER_USERNAME}/functest-vnf:x86_64-latest \
+          --target ${DOCKER_USERNAME}/functest-vnf:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64 \
+          --template ${DOCKER_USERNAME}/functest-restapi:x86_64-latest \
+          --target ${DOCKER_USERNAME}/functest-restapi:latest
+    - script: >
+        sudo manifest-tool push from-args \
+          --platforms linux/amd64 \
+          --template ${DOCKER_USERNAME}/functest-parser:x86_64-latest \
+          --target ${DOCKER_USERNAME}/functest-parser:latest
diff --git a/build.sh b/build.sh
index 18d2e58de..3b46cb06e 100644
--- a/build.sh
+++ b/build.sh
@@ -3,7 +3,7 @@
 set -e
 
 repo=${repo:-opnfv}
-dirs="\
+x86_64_dirs=${x86_64_dirs-"\
 docker/core \
 docker/healthcheck \
 docker/smoke \
@@ -11,14 +11,23 @@ docker/features \
 docker/components \
 docker/vnf \
 docker/parser \
-docker/restapi"
+docker/restapi"}
+aarch64_dirs=${aarch64_dirs-$(echo "${x86_64_dirs}" | sed -e "s|docker/vnf||" \
+    -e "s|docker/restapi||")}
 
-(cd docker && docker build -t "${repo}/functest" .)
-docker push "${repo}/functest"
+find . -name Dockerfile -exec sed -i -e "s|opnfv/functest-core|${repo}/functest-core:x86_64-latest|g" {} +
+for dir in ${x86_64_dirs}; do
+    (cd "${dir}" && docker build -t "${repo}/functest-${dir##**/}:x86_64-latest" .)
+    docker push "${repo}/functest-${dir##**/}:x86_64-latest"
+done
+find . -name Dockerfile -exec git checkout {} +
 
-for dir in ${dirs}; do
-    (cd ${dir} && docker build -t "${repo}/functest-${dir##**/}" .)
-    docker push "${repo}/functest-${dir##**/}"
+find . -name Dockerfile -exec sed -i -e "s|alpine:3.6|multiarch/alpine:aarch64-v3.6|g" {} +
+find . -name Dockerfile -exec sed -i -e "s|opnfv/functest-core|${repo}/functest-core:aarch64-latest|g" {} +
+for dir in ${aarch64_dirs}; do
+    (cd "${dir}" && docker build -t "${repo}/functest-${dir##**/}:aarch64-latest" .)
+    docker push "${repo}/functest-${dir##**/}:aarch64-latest"
 done
+find . -name Dockerfile -exec git checkout {} +
 
 exit $?
-- 
cgit 1.2.3-korg