Clone repositories instead of wgetting constraints

It stops leveraging on wget to get constraints as it fails since
opendev was created. Then functest and requirements are no longer
removed for functest-core which simplifies all child Dockerfile.

It also fix the current incompatibilities in OpenStack constraints
ERROR: cliff 3.2.0 has requirement cmd2!=0.8.3,<0.9.0,>=0.8.0, but you'll have cmd2 1.1.0 which is incompatible.
https://github.com/openstack/requirements/blob/master/upper-constraints.txt#L230
https://github.com/openstack/requirements/blob/master/upper-constraints.txt#L416

Change-Id: I7c044208ef26ad047fb841710b422acb6c9f0320
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit 961b940c796871e40146b90cf19b48390310dd59)

7 files changed, 48 insertions, 72 deletions

diff --git a/docker/benchmarking-cntt/Dockerfile b/docker/benchmarking-cntt/Dockerfile
index ae1d115e0..a7411af81 100644
--- a/docker/benchmarking-cntt/Dockerfile
+++ b/docker/benchmarking-cntt/Dockerfile
@@ -1,5 +1,5 @@
 FROM opnfv/functest-benchmarking:kali
 
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
-COPY blacklist.yaml /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/rally/blacklist.yaml
+COPY blacklist.yaml /src/functest/functest/opnfv_tests/openstack/rally/blacklist.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile
index cc1b5a81e..fa33a82bd 100644
--- a/docker/benchmarking/Dockerfile
+++ b/docker/benchmarking/Dockerfile
@@ -1,7 +1,5 @@
 FROM opnfv/functest-core:kali
 
-ARG BRANCH=stable/kali
-ARG OPENSTACK_TAG=stable/ussuri
 ARG VMTP_TAG=34a82c9f3598ec7f5d8de0a6d5139b92931db4cc
 ARG NEUTRON_TAG=stable/ussuri
 
@@ -9,20 +7,16 @@ RUN apk --no-cache add --update libxml2 libxslt && \
     apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev \
         openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
-    case $(uname -m) in aarch*|arm*) CFLAGS="-O0" pip3 install --no-cache-dir -cupper-constraints.txt -cupper-constraints.opnfv.txt lxml ;; esac && \
+    case $(uname -m) in aarch*|arm*) CFLAGS="-O0" \
+        pip3 install --no-cache-dir -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt lxml ;; esac && \
     git init /src/vmtp && \
     (cd /src/vmtp && \
         git fetch --tags https://review.opendev.org/x/vmtp.git $VMTP_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/vmtp/ && \
-    sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/opendev.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    update-requirements -s --source /src/requirements /src/vmtp/ && \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         /src/vmtp && \
     mkdir -p /home/opnfv/functest/data/rally/neutron && \
     git init /src/neutron && \
@@ -30,7 +24,7 @@ RUN apk --no-cache add --update libxml2 libxslt && \
         git fetch --tags https://opendev.org/openstack/neutron.git $NEUTRON_TAG && \
         git checkout FETCH_HEAD) && \
     cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
-    rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vmtp /src/neutron && \
+    rm -r /src/vmtp /src/neutron && \
     apk del .build-deps
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile
index f2404ab6d..f1b973663 100644
--- a/docker/core/Dockerfile
+++ b/docker/core/Dockerfile
@@ -10,25 +10,27 @@ RUN apk --no-cache add --update \
     apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev \
         openssl-dev libjpeg-turbo-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt && apk add --no-cache py3-pynacl ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt  && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.opnfv.txt -cupper-constraints.txt \
-        -e git+https://opendev.org/openstack/requirements@$OPENSTACK_TAG#egg=openstack_requirements && \
+    git init /src/requirements && \
+    (cd /src/requirements && \
+        git fetch --tags https://review.opendev.org/openstack/requirements $OPENSTACK_TAG && \
+        git checkout FETCH_HEAD) && \
     git init /src/functest && \
     (cd /src/functest && \
         git fetch --tags https://gerrit.opnfv.org/gerrit/functest $BRANCH && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/functest && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.opnfv.txt -cupper-constraints.txt \
-        /src/functest && \
+    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d /src/requirements/upper-constraints.txt && apk add --no-cache py3-pynacl ;; esac && \
+    sed -i -E /#egg=functest/d /src/functest/upper-constraints.txt && \
+    sed -i -E /^tempest==+.*$/d /src/requirements/upper-constraints.txt && \
+    pip3 install --no-cache-dir --src /src -c/src/functest/upper-constraints.txt -c/src/requirements/upper-constraints.txt \
+        -e /src/requirements && \
+    update-requirements -s --source /src/requirements /src/functest && \
+    pip3 install --no-cache-dir --src /src -c/src/functest/upper-constraints.txt -c/src/requirements/upper-constraints.txt \
+        -e /src/functest && \
     (cd /src/rally && patch -p1 < /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch) && \
-    rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/functest \
+    rm -r /src/requirements/.git /src/functest/.git \
         /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch && \
-    cp /usr/lib/python3.8/site-packages/functest/ci/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
-    cp /usr/lib/python3.8/site-packages/functest/ci/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
+    cp /src/functest/functest/ci/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
+    cp /src/functest/functest/ci/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
     bash -c "mkdir -p /var/lib/xtesting /home/opnfv" && \
     ln -s /var/lib/xtesting /home/opnfv/functest && \
     bash -c "mkdir -p /home/opnfv/functest{/conf,/data,/images,/results} /home/opnfv/repos/vnfs" && \
diff --git a/docker/healthcheck/Dockerfile b/docker/healthcheck/Dockerfile
index 14e8fb2d1..f20412efb 100644
--- a/docker/healthcheck/Dockerfile
+++ b/docker/healthcheck/Dockerfile
@@ -1,32 +1,24 @@
 FROM opnfv/functest-core:kali
 
-ARG BRANCH=stable/kali
-ARG OPENSTACK_TAG=stable/ussuri
 ARG ODL_TAG=89b88a0a23561f0bda62338b394ec41655679b2d
 ARG TEMPEST_HORIZON_TAG=1.0.0
 
 COPY thirdparty-requirements.txt thirdparty-requirements.txt
 RUN apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev openssl-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
     git init /src/tempest-horizon && \
     (cd /src/tempest-horizon && \
         git fetch --tags https://opendev.org/openstack/tempest-horizon.git $TEMPEST_HORIZON_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/tempest-horizon/ && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    update-requirements -s --source /src/requirements /src/tempest-horizon/ && \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         /src/tempest-horizon -rthirdparty-requirements.txt && \
     git init /src/odl_test && \
     (cd /src/odl_test && \
         git fetch --tags https://git.opendaylight.org/gerrit/integration/test $ODL_TAG && \
         git checkout FETCH_HEAD) && \
-    rm -r /src/odl_test/.git /src/tempest-horizon/ thirdparty-requirements.txt \
-        upper-constraints.txt upper-constraints.opnfv.txt && \
+    rm -r /src/odl_test/.git /src/tempest-horizon/ thirdparty-requirements.txt && \
     apk del .build-deps
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke-cntt/Dockerfile b/docker/smoke-cntt/Dockerfile
index f92ad25d4..34e532d2f 100644
--- a/docker/smoke-cntt/Dockerfile
+++ b/docker/smoke-cntt/Dockerfile
@@ -1,5 +1,5 @@
 FROM opnfv/functest-smoke:kali
 
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
-COPY tempest_conf.yaml /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml
+COPY tempest_conf.yaml /src/functest/functest/opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile
index e34613415..0f5a0ffd3 100644
--- a/docker/smoke/Dockerfile
+++ b/docker/smoke/Dockerfile
@@ -1,7 +1,5 @@
 FROM opnfv/functest-core:kali
 
-ARG BRANCH=stable/kali
-ARG OPENSTACK_TAG=stable/ussuri
 ARG PATROLE_TAG=0.9.0
 ARG NEUTRON_TEMPEST_TAG=1.1.0
 ARG CINDER_TEMPEST_TAG=0.3.1
@@ -17,68 +15,62 @@ RUN apk --no-cache add --update libxml2 libxslt && \
     apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev \
         openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
-    case $(uname -m) in aarch*|arm*) CFLAGS="-O0" pip3 install --no-cache-dir -cupper-constraints.txt -cupper-constraints.opnfv.txt lxml ;; esac && \
     git init /src/patrole && \
     (cd /src/patrole && \
         git fetch --tags https://opendev.org/openstack/patrole.git $PATROLE_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/patrole/ && \
+    update-requirements -s --source /src/requirements /src/patrole/ && \
     git init /src/neutron-tempest-plugin && \
     (cd /src/neutron-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/neutron-tempest-plugin.git $NEUTRON_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/neutron-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/neutron-tempest-plugin && \
     git init /src/cinder-tempest-plugin && \
     (cd /src/cinder-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/cinder-tempest-plugin.git $CINDER_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/cinder-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/cinder-tempest-plugin && \
     git init /src/keystone-tempest-plugin && \
     (cd /src/keystone-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/keystone-tempest-plugin.git $KEYSTONE_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/keystone-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/keystone-tempest-plugin && \
     git init /src/barbican-tempest-plugin && \
     (cd /src/barbican-tempest-plugin && \
         git fetch --tags https://opendev.org/openstack/barbican-tempest-plugin.git $BARBICAN_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/barbican-tempest-plugin/ && \
+    update-requirements -s --source /src/requirements /src/barbican-tempest-plugin/ && \
     git init /src/octavia-tempest-plugin && \
     (cd /src/octavia-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/octavia-tempest-plugin.git $OCTAVIA_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/octavia-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/octavia-tempest-plugin && \
     git init /src/heat-tempest-plugin && \
     (cd /src/heat-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/heat-tempest-plugin.git $HEAT_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/heat-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/heat-tempest-plugin && \
     git init /src/telemetry-tempest-plugin && \
     (cd /src/telemetry-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/telemetry-tempest-plugin.git $TELEMETRY_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/telemetry-tempest-plugin && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    update-requirements -s --source /src/requirements /src/telemetry-tempest-plugin && \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
         /src/cinder-tempest-plugin /src/keystone-tempest-plugin \
         /src/octavia-tempest-plugin /src/heat-tempest-plugin /src/telemetry-tempest-plugin && \
     mkdir -p /home/opnfv/functest/data/refstack && \
     mkdir -p /etc/neutron /etc/glance && \
     virtualenv --no-pip --no-setuptools --no-wheel oslo && . oslo/bin/activate && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         oslo.policy -e git+https://opendev.org/openstack/neutron.git@$NEUTRON_TAG#egg=neutron \
         -e git+https://opendev.org/openstack/glance.git@$GLANCE_TAG#egg=glance && \
     oslopolicy-sample-generator  --format json --output-file /etc/neutron/policy.json --namespace neutron && \
     oslopolicy-sample-generator  --format json --output-file /etc/glance/policy.json --namespace glance && \
     deactivate && \
-    rm -r oslo upper-constraints.txt upper-constraints.opnfv.txt \
-        /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
+    rm -r oslo /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
         /src/cinder-tempest-plugin /src/keystone-tempest-plugin \
         /src/octavia-tempest-plugin /src/heat-tempest-plugin \
         /src/telemetry-tempest-plugin /src/neutron /src/glance && \
diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile
index 63f385135..cf50f0980 100644
--- a/docker/vnf/Dockerfile
+++ b/docker/vnf/Dockerfile
@@ -1,7 +1,5 @@
 FROM opnfv/functest-core:kali
 
-ARG BRANCH=stable/kali
-ARG OPENSTACK_TAG=stable/ussuri
 ARG VIMS_TEST_TAG=release-130
 ARG QUAFF_TAG=59213d6d8ee29433552bb75f505cdc96b0b18909
 ARG CLOUDIFY_VIMS_TAG=gambia
@@ -23,13 +21,10 @@ RUN apk --no-cache add --update \
         procps libxslt libxml2 zlib libffi go musl-dev && \
     apk --no-cache add --virtual .build-deps --update \
         ruby-dev g++ make libxslt-dev libxml2-dev zlib-dev libffi-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
-    git clone --depth 1 -b $VIMS_TEST_TAG https://github.com/Metaswitch/clearwater-live-test /src/vims-test && \
+    git init /src/vims-test && \
+    (cd /src/vims-test && \
+        git fetch --tags https://github.com/Metaswitch/clearwater-live-test $VIMS_TEST_TAG && \
+        git checkout FETCH_HEAD) && \
     sed -i s/unf_ext\ \(.*\)/unf_ext\ \(0.0.7.4\)/g /src/vims-test/Gemfile.lock && \
     git init /src/vims-test/quaff && \
     (cd /src/vims-test/quaff && \
@@ -64,7 +59,8 @@ RUN apk --no-cache add --update \
         (cd /src/epc-requirements/abot_charm && \
             git fetch --tags https://github.com/RebacaInc/abot_charm.git $ABOT_CHARM && \
             git checkout FETCH_HEAD) && \
-        pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+        pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+            -c/src/functest/upper-constraints.txt \
             juju-wait==$JUJU_WAIT_TAG && \
         go get -d github.com/rogpeppe/godeps && \
         (cd $GOPATH/src/github.com/rogpeppe/godeps && git checkout $GODEPS_TAG && go install -v github.com/rogpeppe/godeps) && \
@@ -74,7 +70,7 @@ RUN apk --no-cache add --update \
         rm -r $GOPATH/src/ $GOPATH/pkg /src/epc-requirements/abot_charm/.git /root/.cache/go-build;; \
     esac && \
     (cd /src/vims-test && bundle config build.nokogiri --use-system-libraries && bundle install --system && bundle update rest-client) && \
-    rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \
+    rm -r /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \
         /src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \
         /tmp/clearwater-heat-singlenet-deps.patch && \
     apk del .build-deps