Update Alpine to 3.11

It also updates Python to 3.8 (default version).
It selects arm32v6 et arm64v8 repos and switches multiarch binfmt.

https://alpinelinux.org/posts/Alpine-3.11.0-released.html

Change-Id: Ib41cf0ab805ce2137658f3ff656b0a88c7b4ebf5
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

19 files changed, 51 insertions, 48 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2b5a0a78d..a5c8346ee 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,7 +3,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:3.7.4
+      - image: circleci/python:3.8
     steps:
       - checkout
       - run:
diff --git a/.travis.yml b/.travis.yml
index 122a602a7..0810c6272 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,10 +9,10 @@ addons:
       - libssl1.0.0
 
 before_script:
-  - sudo apt-get -y install qemu-user-static enchant
+  - sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
   - sudo add-apt-repository -y ppa:deadsnakes/ppa
   - sudo apt-get update
-  - sudo apt-get install python3.7 python3.7-dev
+  - sudo apt-get install python3.8 python3.8-dev python3.8-distutils enchant
   - sudo pip install tox
   - sudo -E docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
   - (cd .. && git clone https://github.com/estesp/manifest-tool)
@@ -24,7 +24,7 @@ jobs:
     - stage: run unit tests
       script: >
         tox -e \
-          docs,pep8,pylint,yamllint,ansiblelint,bashate,bandit,py27,py37,cover
+          docs,pep8,pylint,yamllint,ansiblelint,bashate,bandit,py27,py38,cover
     - stage: build functest-core images
       script: sudo -E bash build.sh
       env:
diff --git a/ansible/site.cntt.yml b/ansible/site.cntt.yml
index 6eb2a4dba..604b98e97 100644
--- a/ansible/site.cntt.yml
+++ b/ansible/site.cntt.yml
@@ -10,7 +10,7 @@
           - repo: _
             dport:
             container: alpine
-            tag: '3.10'
+            tag: '3.11'
         steps:
           - name: build opnfv/functest-core
             containers:
diff --git a/ansible/site.yml b/ansible/site.yml
index e29f208a7..96ff8d07d 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -10,7 +10,7 @@
           - repo: _
             dport:
             container: alpine
-            tag: '3.10'
+            tag: '3.11'
         steps:
           - name: build opnfv/functest-core
             containers:
diff --git a/build.sh b/build.sh
index f13c623ee..d6726d320 100644
--- a/build.sh
+++ b/build.sh
@@ -34,11 +34,11 @@ for dir in ${amd64_dirs}; do
         (docker rmi "${repo}/functest-${dir##**/}:amd64-latest" || true)
 done
 [ -n "${amd64_dirs}" ] &&
-    (docker rmi "${repo}/functest-core:amd64-latest" alpine:3.10 || true)
+    (docker rmi "${repo}/functest-core:amd64-latest" alpine:3.11 || true)
 find . -name Dockerfile -exec git checkout {} +
 
 find . -name Dockerfile -exec sed -i \
-    -e "s|alpine:3.10|multiarch/alpine:arm64-v3.10|g" {} +
+    -e "s|alpine:3.11|arm64v8/alpine:3.11|g" {} +
 find . -name Dockerfile -exec sed -i \
     -e "s|opnfv/functest-core|${repo}/functest-core:arm64-latest|g" {} +
 find . -name Dockerfile -exec sed -i \
@@ -52,11 +52,11 @@ for dir in ${arm64_dirs}; do
 done
 [ -n "${arm64_dirs}" ] &&
     (docker rmi "${repo}/functest-core:arm64-latest" \
-        multiarch/alpine:arm64-v3.10 || true)
+        arm64v8/alpine:3.11 || true)
 find . -name Dockerfile -exec git checkout {} +
 
 find . -name Dockerfile -exec sed -i \
-    -e "s|alpine:3.10|multiarch/alpine:armhf-v3.10|g" {} +
+    -e "s|alpine:3.11|arm32v6/alpine:3.11|g" {} +
 find . -name Dockerfile -exec sed -i \
     -e "s|opnfv/functest-core|${repo}/functest-core:arm-latest|g" {} +
 find . -name Dockerfile -exec sed -i \
@@ -70,7 +70,7 @@ for dir in ${arm_dirs}; do
 done
 [ -n "${arm_dirs}" ] &&
     (docker rmi "${repo}/functest-core:arm-latest" \
-        multiarch/alpine:armhf-v3.10 || true)
+        arm32v6/alpine:3.11 || true)
 find . -name Dockerfile -exec git checkout {} +
 
 exit $?
diff --git a/docker/benchmarking-cntt/Dockerfile b/docker/benchmarking-cntt/Dockerfile
index 56641e352..59fe5a88d 100644
--- a/docker/benchmarking-cntt/Dockerfile
+++ b/docker/benchmarking-cntt/Dockerfile
@@ -1,4 +1,4 @@
 FROM opnfv/functest-benchmarking
 
-COPY testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
+COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile
index a282886f2..f53f22b10 100644
--- a/docker/benchmarking/Dockerfile
+++ b/docker/benchmarking/Dockerfile
@@ -30,5 +30,5 @@ RUN apk --no-cache add --update libxml2 libxslt && \
     cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
     rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vmtp /src/neutron && \
     apk del .build-deps
-COPY testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
+COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile
index 7bf9297a2..7d66e3b10 100644
--- a/docker/core/Dockerfile
+++ b/docker/core/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.10
+FROM alpine:3.11
 
 ARG BRANCH=master
 ARG OPENSTACK_TAG=master
@@ -25,8 +25,8 @@ RUN apk --no-cache add --update \
     pip3 install --no-cache-dir --src /src -cupper-constraints.opnfv.txt -cupper-constraints.txt \
         /src/functest && \
     rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/functest && \
-    cp /usr/lib/python3.7/site-packages/functest/ci/logging.ini /usr/lib/python3.7/site-packages/xtesting/ci/ && \
-    cp /usr/lib/python3.7/site-packages/functest/ci/logging.debug.ini /usr/lib/python3.7/site-packages/xtesting/ci/ && \
+    cp /usr/lib/python3.8/site-packages/functest/ci/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
+    cp /usr/lib/python3.8/site-packages/functest/ci/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
     bash -c "mkdir -p /var/lib/xtesting /home/opnfv" && \
     ln -s /var/lib/xtesting /home/opnfv/functest && \
     bash -c "mkdir -p /home/opnfv/functest{/conf,/data,/images,/results} /home/opnfv/repos/vnfs" && \
diff --git a/docker/healthcheck/Dockerfile b/docker/healthcheck/Dockerfile
index b9e126790..885d501b2 100644
--- a/docker/healthcheck/Dockerfile
+++ b/docker/healthcheck/Dockerfile
@@ -27,5 +27,5 @@ RUN apk --no-cache add --virtual .build-deps --update \
     rm -r /src/odl_test/.git /src/tempest-horizon/ thirdparty-requirements.txt \
         upper-constraints.txt upper-constraints.opnfv.txt && \
     apk del .build-deps
-COPY testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
+COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke-cntt/Dockerfile b/docker/smoke-cntt/Dockerfile
index 4c78ecb1b..c6b097093 100644
--- a/docker/smoke-cntt/Dockerfile
+++ b/docker/smoke-cntt/Dockerfile
@@ -1,5 +1,5 @@
 FROM opnfv/functest-smoke
 
-COPY testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
-COPY tempest_conf.yaml /usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml
+COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
+COPY tempest_conf.yaml /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile
index e62a35076..1467dd615 100644
--- a/docker/smoke/Dockerfile
+++ b/docker/smoke/Dockerfile
@@ -71,5 +71,5 @@ RUN apk --no-cache add --update libxml2 libxslt && \
 COPY compute.txt /home/opnfv/functest/data/refstack/compute.txt
 COPY object.txt /home/opnfv/functest/data/refstack/object.txt
 COPY platform.txt /home/opnfv/functest/data/refstack/platform.txt
-COPY testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
+COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile
index cf67c8a77..dc5290de0 100644
--- a/docker/vnf/Dockerfile
+++ b/docker/vnf/Dockerfile
@@ -77,5 +77,5 @@ RUN apk --no-cache add --update \
         /src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \
         /tmp/clearwater-heat-singlenet-deps.patch && \
     apk del .build-deps
-COPY testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
+COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst
index f0583f245..131bf1191 100644
--- a/docs/testing/user/configguide/configguide.rst
+++ b/docs/testing/user/configguide/configguide.rst
@@ -353,7 +353,7 @@ By default all the logs are put un /home/opnfv/functest/results/functest.log.
 If you want to have more logs in console, you may edit the logging.ini file
 manually.
 Connect on the docker then edit the file located in
-/usr/lib/python3.7/site-packages/xtesting/ci/logging.ini
+/usr/lib/python3.8/site-packages/xtesting/ci/logging.ini
 
 Change wconsole to console in the desired module to get more traces.
 
@@ -363,8 +363,8 @@ Configuration
 
 You may also directly modify the python code or the configuration file (e.g.
 testcases.yaml used to declare test constraints) under
-/usr/lib/python3.7/site-packages/xtesting and
-/usr/lib/python3.7/site-packages/functest
+/usr/lib/python3.8/site-packages/xtesting and
+/usr/lib/python3.8/site-packages/functest
 
 
 Tips
diff --git a/docs/testing/user/userguide/test_results.rst b/docs/testing/user/userguide/test_results.rst
index a1d4dfb59..5f78c1dc2 100644
--- a/docs/testing/user/userguide/test_results.rst
+++ b/docs/testing/user/userguide/test_results.rst
@@ -10,7 +10,7 @@ In manual mode test results are displayed in the console and result files
 are put in /home/opnfv/functest/results.
 
 If you want additional logs, you may configure the logging.ini under
-/usr/lib/python3.7/site-packages/xtesting/ci.
+/usr/lib/python3.8/site-packages/xtesting/ci.
 
 Automated testing
 -----------------
diff --git a/docs/testing/user/userguide/troubleshooting.rst b/docs/testing/user/userguide/troubleshooting.rst
index 66e2433f6..b13853c48 100644
--- a/docs/testing/user/userguide/troubleshooting.rst
+++ b/docs/testing/user/userguide/troubleshooting.rst
@@ -44,8 +44,8 @@ follows::
 The Functest CLI is designed to route a call to the corresponding internal
 python scripts, located in paths::
 
-    /usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/vping/vping_ssh.py
-    /usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/vping/vping_userdata.py
+    /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/vping/vping_ssh.py
+    /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/vping/vping_userdata.py
 
 Notes:
 
@@ -125,7 +125,7 @@ This test case creates a floating IP on the external network and assigns it to
 the second instance **opnfv-vping-2**. The purpose of this is to establish
 a SSH connection to that instance and SCP a script that will ping the first
 instance. This script is located in the repository under
-/usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/vping/ping.sh
+/usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/vping/ping.sh
 and takes an IP as a parameter. When the SCP is completed, the test will do a
 SSH call to that script inside the second instance. Some problems can happen
 here::
@@ -278,7 +278,7 @@ To enable that, add a new entry in docker/smoke/testcases.yaml on the
 
 Also, a list of the Tempest test cases must be provided to the container or
 modify the existing one in
-/usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/test_list.txt
+/usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/test_list.txt
 
 Example of custom list of tests 'my-custom-tempest-tests.txt'::
 
@@ -290,8 +290,8 @@ This is an example of running a customized list of Tempest tests in Functest::
   sudo docker run --env-file env \
       -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \
       -v $(pwd)/images:/home/opnfv/functest/images \
-      -v $(pwd)/my-custom-testcases.yaml:/usr/lib/python3.7/site-packages/functest/ci/testcases.yaml \
-      -v $(pwd)/my-custom-tempest-tests.txt:/usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/test_list.txt \
+      -v $(pwd)/my-custom-testcases.yaml:/usr/lib/python3.8/site-packages/functest/ci/testcases.yaml \
+      -v $(pwd)/my-custom-tempest-tests.txt:/usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/test_list.txt \
       opnfv/functest-smoke run_tests -t tempest_custom
 
 
@@ -314,7 +314,7 @@ Possible scenarios are:
 
 To know more about what those scenarios are doing, they are defined in
 directory:
-/usr/lib/python3.7/site-packages/functest/opnfv_tests/openstack/rally/scenario
+/usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/rally/scenario
 For more info about Rally scenario definition please refer to the Rally
 official documentation. `[3]`_
 
diff --git a/functest/opnfv_tests/openstack/vping/vping_userdata.py b/functest/opnfv_tests/openstack/vping/vping_userdata.py
index 225c167d5..9010895cb 100644
--- a/functest/opnfv_tests/openstack/vping/vping_userdata.py
+++ b/functest/opnfv_tests/openstack/vping/vping_userdata.py
@@ -79,13 +79,13 @@ class VPingUserdata(singlevm.VmReady2):
                 self.logger.info("vPing detected!")
                 exit_code = testcase.TestCase.EX_OK
                 break
-            elif "failed to read iid from metadata" in p_console or tries > 5:
+            if "failed to read iid from metadata" in p_console or tries > 5:
                 self.logger.info("Failed to read iid from metadata")
                 break
-            elif sec == getattr(config.CONF, 'vping_ping_timeout'):
+            if sec == getattr(config.CONF, 'vping_ping_timeout'):
                 self.logger.info("Timeout reached.")
                 break
-            elif sec % 10 == 0:
+            if sec % 10 == 0:
                 if "request failed" in p_console:
                     self.logger.debug(
                         "It seems userdata is not supported in nova boot. "
diff --git a/functest/utils/functest_utils.py b/functest/utils/functest_utils.py
index a1434adc2..a9c9e5e72 100644
--- a/functest/utils/functest_utils.py
+++ b/functest/utils/functest_utils.py
@@ -207,7 +207,7 @@ def convert_ini_to_dict(value):
     "Convert oslo.conf input to dict"
     assert isinstance(value, str)
     try:
-        return {k: v for k, v in (x.rsplit(':', 1) for x in value.split(','))}
+        return dict((x.rsplit(':', 1) for x in value.split(',')))
     except ValueError:
         return {}
 
@@ -217,4 +217,4 @@ def convert_ini_to_list(value):
     assert isinstance(value, str)
     if not value:
         return []
-    return [x for x in value.split(',')]
+    return list(value.split(','))
diff --git a/tox.ini b/tox.ini
index cc712791f..f02ff3a15 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = docs,pep8,pylint,yamllint,ansiblelint,bashate,bandit,py27,py37,cover,perm
+envlist = docs,pep8,pylint,yamllint,ansiblelint,bashate,bandit,py27,py38,cover,perm
 
 [testenv]
 usedevelop = True
@@ -18,7 +18,7 @@ commands = nosetests --with-xunit \
   functest/tests/unit
 
 [testenv:docs]
-basepython = python3.7
+basepython = python3.8
 commands =
   doc8 \
     --ignore-path api/build \
@@ -32,18 +32,18 @@ commands =
   sphinx-build -W -b spelling -Dextensions=sphinxcontrib.spelling docs docs/build/spellcheck
 
 [testenv:pep8]
-basepython = python3.7
+basepython = python3.8
 commands = flake8
 
 [testenv:pylint]
-basepython = python3.7
+basepython = python3.8
 commands =
   pylint \
     --ignore-imports=y --min-similarity-lines=10 \
     --disable=locally-disabled functest
 
 [testenv:yamllint]
-basepython = python3.7
+basepython = python3.8
 files =
   .travis.yml
   docker
@@ -57,7 +57,7 @@ commands =
   yamllint -s {[testenv:yamllint]files}
 
 [testenv:ansiblelint]
-basepython = python3.7
+basepython = python3.8
 commands =
   ansible-lint -x303 ansible/site.yml
 
@@ -65,7 +65,7 @@ commands =
 commands = nosetests functest/tests/unit
 
 [testenv:bashate]
-basepython = python3.7
+basepython = python3.8
 files =
   functest/opnfv_tests/openstack/cinder/write_data.sh
   functest/opnfv_tests/openstack/cinder/read_data.sh
@@ -77,11 +77,11 @@ commands = bashate -e E005,E006,E042,E043 {[testenv:bashate]files}
 
 
 [testenv:bandit]
-basepython = python3.7
+basepython = python3.8
 commands = bandit -r functest -x tests -n 5 -ll -s B601,B602
 
 [testenv:cover]
-basepython = python3.7
+basepython = python3.8
 dirs =
   functest/tests/unit/odl
   functest/tests/unit/openstack/vping
@@ -94,7 +94,7 @@ commands = nosetests --with-coverage --cover-tests \
   --cover-min-percentage 100 {[testenv:cover]dirs}
 
 [testenv:perm]
-basepython = python3.7
+basepython = python3.8
 whitelist_externals = bash
 path=. -not -path './.tox/*' -not -path './.git/*' -not -path './docs/com/pres/reveal.js/*'
 commands =
diff --git a/upper-constraints.txt b/upper-constraints.txt
index 212d1454d..7fa6fa226 100644
--- a/upper-constraints.txt
+++ b/upper-constraints.txt
@@ -16,6 +16,7 @@ robotframework-requests===0.5.0
 robotframework-sshlibrary===3.3.0
 ansible===2.9.2
 xtesting===0.86.0
+git+https://github.com/PyCQA/bandit@3d0824676974e7e2e9635c10bc4f12e261f1dbdf#egg=bandit
 bandit===1.1.0
 ruamel.yaml.jinja2==0.2.2
 -e git+https://opendev.org/openstack/tempest#egg=tempest
@@ -24,6 +25,8 @@ git+https://opendev.org/openstack/rally-openstack.git@e62608d35302ed3c0aeb31784c
 git+https://github.com/xrally/xrally-kubernetes.git#egg=xrally-kubernetes
 git+https://github.com/esnme/ultrajson@d25e024f481c5571d15f3c0c406a498ca0467cfd#egg=ujson
 pylint===1.9.5;python_version=='2.7'
-pylint===2.3.1;python_version=='3.7'
+pylint===2.4.4;python_version=='3.8'
+flake8===3.7.9
+nose===1.3.7
 ruamel.yaml===0.15.100
 sphinxcontrib-spelling===4.3.0