diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | .travis.yml | 133 | ||||
| -rw-r--r-- | build.sh | 23 | ||||
| -rw-r--r-- | docs/com/pres/dockerslicing/dockerslicing.md | 170 | ||||
| -rw-r--r-- | docs/com/pres/dockerslicing/index.html | 52 |
5 files changed, 372 insertions, 7 deletions
diff --git a/.gitignore b/.gitignore index f6e406203..81ebca89c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ functest.egg-info *venv .tox *.out +*.pdf diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..af67f4786 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,133 @@ +sudo: required +language: bash +services: docker +language: generic + +before_script: + - sudo apt-get -y install qemu-user-static + - sudo -E docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}" + - (cd .. && git clone https://github.com/estesp/manifest-tool) + - (cd ../manifest-tool && sudo -E make && sudo -E make install) + +jobs: + include: + - stage: build functest-core images + script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="" + - aarch64_dirs="docker/core" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/core" + - aarch64_dirs="" + - stage: publish functest-core manifests + script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${DOCKER_USERNAME}/functest-core:x86_64-latest \ + --template ${DOCKER_USERNAME}/functest-core:aarch64-latest \ + --target ${DOCKER_USERNAME}/functest-core:latest + - stage: build all functest images + script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="" + - aarch64_dirs="docker/healthcheck" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="" + - aarch64_dirs="docker/smoke" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="" + - aarch64_dirs="docker/features" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="" + - aarch64_dirs="docker/components" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/healthcheck" + - aarch64_dirs="" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/smoke" + - aarch64_dirs="" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/features" + - aarch64_dirs="" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/components" + - aarch64_dirs="" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/vnf" + - aarch64_dirs="" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/restapi" + - aarch64_dirs="" + - script: sudo -E bash build.sh + env: + - repo="${DOCKER_USERNAME}" + - x86_64_dirs="docker/parser" + - aarch64_dirs="" + - stage: publish all manifests + script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${DOCKER_USERNAME}/functest-healthcheck:x86_64-latest \ + --template ${DOCKER_USERNAME}/functest-healthcheck:aarch64-latest \ + --target ${DOCKER_USERNAME}/functest-healthcheck:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${DOCKER_USERNAME}/functest-healthcheck:x86_64-latest \ + --template ${DOCKER_USERNAME}/functest-healthcheck:aarch64-latest \ + --target ${DOCKER_USERNAME}/functest-healthcheck:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${DOCKER_USERNAME}/functest-smoke:x86_64-latest \ + --template ${DOCKER_USERNAME}/functest-smoke:aarch64-latest \ + --target ${DOCKER_USERNAME}/functest-smoke:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${DOCKER_USERNAME}/functest-features:x86_64-latest \ + --template ${DOCKER_USERNAME}/functest-features:aarch64-latest \ + --target ${DOCKER_USERNAME}/functest-features:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${DOCKER_USERNAME}/functest-components:x86_64-latest \ + --template ${DOCKER_USERNAME}/functest-components:aarch64-latest \ + --target ${DOCKER_USERNAME}/functest-components:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64 \ + --template ${DOCKER_USERNAME}/functest-vnf:x86_64-latest \ + --target ${DOCKER_USERNAME}/functest-vnf:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64 \ + --template ${DOCKER_USERNAME}/functest-restapi:x86_64-latest \ + --target ${DOCKER_USERNAME}/functest-restapi:latest + - script: > + sudo manifest-tool push from-args \ + --platforms linux/amd64 \ + --template ${DOCKER_USERNAME}/functest-parser:x86_64-latest \ + --target ${DOCKER_USERNAME}/functest-parser:latest @@ -3,7 +3,7 @@ set -e repo=${repo:-opnfv} -dirs="\ +x86_64_dirs=${x86_64_dirs-"\ docker/core \ docker/healthcheck \ docker/smoke \ @@ -11,14 +11,23 @@ docker/features \ docker/components \ docker/vnf \ docker/parser \ -docker/restapi" +docker/restapi"} +aarch64_dirs=${aarch64_dirs-$(echo "${x86_64_dirs}" | sed -e "s|docker/vnf||" \ + -e "s|docker/restapi||")} -(cd docker && docker build -t "${repo}/functest" .) -docker push "${repo}/functest" +find . -name Dockerfile -exec sed -i -e "s|opnfv/functest-core|${repo}/functest-core:x86_64-latest|g" {} + +for dir in ${x86_64_dirs}; do + (cd "${dir}" && docker build -t "${repo}/functest-${dir##**/}:x86_64-latest" .) + docker push "${repo}/functest-${dir##**/}:x86_64-latest" +done +find . -name Dockerfile -exec git checkout {} + -for dir in ${dirs}; do - (cd ${dir} && docker build -t "${repo}/functest-${dir##**/}" .) - docker push "${repo}/functest-${dir##**/}" +find . -name Dockerfile -exec sed -i -e "s|alpine:3.6|multiarch/alpine:aarch64-v3.6|g" {} + +find . -name Dockerfile -exec sed -i -e "s|opnfv/functest-core|${repo}/functest-core:aarch64-latest|g" {} + +for dir in ${aarch64_dirs}; do + (cd "${dir}" && docker build -t "${repo}/functest-${dir##**/}:aarch64-latest" .) + docker push "${repo}/functest-${dir##**/}:aarch64-latest" done +find . -name Dockerfile -exec git checkout {} + exit $? diff --git a/docs/com/pres/dockerslicing/dockerslicing.md b/docs/com/pres/dockerslicing/dockerslicing.md new file mode 100644 index 000000000..55fb2557a --- /dev/null +++ b/docs/com/pres/dockerslicing/dockerslicing.md @@ -0,0 +1,170 @@ +# Docker slicing + +[Cédric Ollivier](mailto:cedric.ollivier@orange.com) + +OPNVF Summit 2017 (Beijing) + + + +## Current issues + + +### Dockerfile + +- it copies all the files hosted by the third-party projects (eg docs, .git...) +- several requirements are downgraded/upgraded when building the container as they are managed one after the other +- it could download packages from [PyPI](https://pypi.python.org/pypi) (e.g. [rally](https://pypi.python.org/pypi/rally/)...) instead of cloning git repository +- build dependencies can't be removed to save space as it creates multiple layers (>70) + + +### setup.py + +- no requirement is installed when calling *python setup.py install* as none of the next keys are set: + - install_requires + - tests_require + - dependency_links +- shell scripts are not installed neither in $PATH nor in dist-packages + + +### setup.py + +- master is considered as an incorrect version +- empty package_data seems useless +- zip_safe flag could be set as False as \__file__ would prevent functest from working in a zipfile +- py_modules should not list cli_base as it was moved into functest package + + + +## Proposals + + +### why not relying on [pbr](https://docs.openstack.org/developer/pbr/)? + +- pbr injects requirements into the install_requires, tests_require and/or dependency_links arguments to setup +- it supports conditional dependencies which can be added to the requirements (e.g. subprocess32; python_version=='2.7') + + +### setup.py + +```python +#!/usr/bin/env python + +from setuptools import setup + +setup( + setup_requires=['pbr>=1.9', 'setuptools>=17.1'], + pbr=True, +) +``` + + +### setup.cfg + +```ini +[metadata] +name = functest +version = 5 +home-page = https://wiki.opnfv.org/display/functest + +[files] +packages = functest +scripts = + docker/docker_remote_api/enable_remote_api.sh + docker/add_images.sh + docker/config_install_env.sh + +[entry_points] +console_scripts = + functest = functest.cli.cli_base:cli +``` + + +### Split requirements into (at least) 3 files + +- **requirements.txt** which could list all abstract (i.e. [not associated with any particular index](https://packaging.python.org/requirements/)) dependencies of the functest package +- **test-requirements.txt** which could list all abstract dependencies required for testing the functest package +- **thirdparty-requirements.txt** which could list all abstract and concrete dependencies required by the full functest docker container + + +### tox.ini + +```ini +[testenv] +usedevelop = True +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + git+https://gerrit.opnfv.org/gerrit/releng#egg=opnfv&subdirectory=modules + git+https://gerrit.opnfv.org/gerrit/barometer#egg=baro_tests + git+https://gerrit.opnfv.org/gerrit/snaps#egg=snaps + +``` + + +### Dockerfile + +``` +RUN pip install \ + git+https://gerrit.opnfv.org/gerrit/functest@$BRANCH#egg=functest \ + git+https://gerrit.opnfv.org/gerrit/releng@$BRANCH#egg=opnfv\&subdirectory=modules \ + git+https://gerrit.opnfv.org/gerrit/barometer@$BRANCH#egg=baro_tests \ + git+https://gerrit.opnfv.org/gerrit/snaps@$BRANCH#egg=snaps +``` + + +### Switch to Alpine + +``` +FROM alpine:3.5 + +RUN apk --no-cache add --update \ + python libffi libssl1.0 libjpeg-turbo py-pip && \ + apk --no-cache add --virtual .build-deps --update \ + python-dev build-base linux-headers libffi-dev \ + openssl-dev libjpeg-turbo-dev git && \ + pip install git+https://gerrit.opnfv.org/gerrit/releng#egg=opnfv\&subdirectory=modules \ + git+https://gerrit.opnfv.org/gerrit/barometer#egg=baro_tests \ + git+https://gerrit.opnfv.org/gerrit/snaps#egg=snaps \ + git+https://gerrit.opnfv.org/gerrit/functest@$BRANCH#egg=functest && \ + apk del .build-deps +``` + + +### Status + + - the change [**"leveraging on pbr"**](https://gerrit.opnfv.org/gerrit/#/c/35813/) can be merged + - docker containers have been published (any test is welcome): + - [ollivier/functest-pbr](https://hub.docker.com/r/ollivier/functest-pbr/) + - [ollivier/functest-alpine](https://hub.docker.com/r/ollivier/functest-alpine/) + - the Dockerfile switching to Alpine can be published too + + + +## Next steps + + +### docker slicing + +the previous alpine example could be the base image and we could simply create other images from it (see [FROM](https://docs.docker.com/engine/reference/builder/#from)) by installing all third party python packages + + +### PyPI + +functest could be published to the [Python Package Index](https://pypi.python.org/pypi) and then we would stop listing urls + + + +## on the road + + +### We could be faced with issues + +- several test cases can fail if their requirements are incomplete in setup.py (we should help them to fix it) +- all conditions for requirements may not be compatible (OpenStack requirements as a guideline?) +- we should fix lots of hardcoded paths hidden by the previous design +- some files could be outside the python package (e.g. functest ci scripts) + + + +## Thank You! + diff --git a/docs/com/pres/dockerslicing/index.html b/docs/com/pres/dockerslicing/index.html new file mode 100644 index 000000000..1c65e4fa0 --- /dev/null +++ b/docs/com/pres/dockerslicing/index.html @@ -0,0 +1,52 @@ +<html> +<head> +<title>Docker slicing</title> +<meta name="author" content="Cédric Ollivier"> +<meta name="viewport" + content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"> +<link rel="stylesheet" href="../reveal.js/css/reveal.css"> +<link rel="stylesheet" href="../reveal.js/css/theme/white.css"> +<link rel="stylesheet" href="../reveal.js/lib/css/zenburn.css"> +<script> +var link = document.createElement( 'link' ); +link.rel = 'stylesheet'; +link.type = 'text/css'; +link.href = window.location.search.match( /print-pdf/gi ) ? '../reveal.js/css/print/pdf.css' : '../reveal.js/css/print/paper.css'; +document.getElementsByTagName( 'head' )[0].appendChild( link ); +</script> +</head> +<body> + <div class="reveal"> + <div class="slides"> + <section data-markdown="dockerslicing.md" data-separator="^\n\n\n" + data-separator-vertical="^\n\n" data-separator-notes="^Note:"></section> + </div> + </div> + <script src="../reveal.js/lib/js/head.min.js"></script> + <script src="../reveal.js/js/reveal.js"></script> + <script> + Reveal.initialize({ + dependencies : [ { + src : '../reveal.js/plugin/markdown/marked.js', + condition : function() { + return !!document.querySelector('[data-markdown]'); + } + }, { + src : '../reveal.js/plugin/markdown/markdown.js', + condition : function() { + return !!document.querySelector('[data-markdown]'); + } + }, { + src: '../reveal.js/plugin/highlight/highlight.js', + async: true, + callback: function() { + hljs.initHighlightingOnLoad(); + } + }, { + src: '../reveal.js/plugin/notes/notes.js', + async: true + } ] + }); + </script> +</body> +</html> |