Clone repositories instead of wgetting constraints

It stops leveraging on wget to get constraints as it fails since
opendev was created. Then functest and requirements are no longer
removed for functest-core which simplifies all child Dockerfile.

It also fix the current incompatibilities in OpenStack constraints
ERROR: cliff 3.2.0 has requirement cmd2!=0.8.3,<0.9.0,>=0.8.0, but you'll have cmd2 1.1.0 which is incompatible.
https://github.com/openstack/requirements/blob/master/upper-constraints.txt#L230
https://github.com/openstack/requirements/blob/master/upper-constraints.txt#L416

Change-Id: I7c044208ef26ad047fb841710b422acb6c9f0320
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

7 files changed, 49 insertions, 73 deletions

diff --git a/docker/benchmarking-cntt/Dockerfile b/docker/benchmarking-cntt/Dockerfile
index 2f4e0dbee..56b69e270 100644
--- a/docker/benchmarking-cntt/Dockerfile
+++ b/docker/benchmarking-cntt/Dockerfile
@@ -1,5 +1,5 @@
 FROM opnfv/functest-benchmarking
 
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
-COPY blacklist.yaml /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/rally/blacklist.yaml
+COPY blacklist.yaml /src/functest/functest/opnfv_tests/openstack/rally/blacklist.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile
index c85ddf643..4d18c34f1 100644
--- a/docker/benchmarking/Dockerfile
+++ b/docker/benchmarking/Dockerfile
@@ -1,7 +1,5 @@
 FROM opnfv/functest-core
 
-ARG BRANCH=master
-ARG OPENSTACK_TAG=master
 ARG VMTP_TAG=master
 ARG NEUTRON_TAG=master
 
@@ -9,20 +7,16 @@ RUN apk --no-cache add --update libxml2 libxslt && \
     apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev \
         openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
-    case $(uname -m) in aarch*|arm*) CFLAGS="-O0" pip3 install --no-cache-dir -cupper-constraints.txt -cupper-constraints.opnfv.txt lxml ;; esac && \
+    case $(uname -m) in aarch*|arm*) CFLAGS="-O0" \
+        pip3 install --no-cache-dir -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt lxml ;; esac && \
     git init /src/vmtp && \
     (cd /src/vmtp && \
         git fetch --tags https://review.opendev.org/x/vmtp.git $VMTP_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/vmtp/ && \
-    sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/opendev.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    update-requirements -s --source /src/requirements /src/vmtp/ && \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         /src/vmtp && \
     mkdir -p /home/opnfv/functest/data/rally/neutron && \
     git init /src/neutron && \
@@ -30,7 +24,7 @@ RUN apk --no-cache add --update libxml2 libxslt && \
         git fetch --tags https://opendev.org/openstack/neutron.git $NEUTRON_TAG && \
         git checkout FETCH_HEAD) && \
     cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
-    rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vmtp /src/neutron && \
+    rm -r /src/vmtp /src/neutron && \
     apk del .build-deps
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile
index 502e87f93..27e2fe1d2 100644
--- a/docker/core/Dockerfile
+++ b/docker/core/Dockerfile
@@ -1,7 +1,7 @@
 FROM alpine:3.12
 
 ARG BRANCH=master
-ARG OPENSTACK_TAG=master
+ARG OPENSTACK_TAG=refs/changes/53/735453/1
 
 COPY Switch-to-threading.Thread-for-Rally-tasks.patch /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch
 RUN apk --no-cache add --update \
@@ -10,25 +10,27 @@ RUN apk --no-cache add --update \
     apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev \
         openssl-dev libjpeg-turbo-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt && apk add --no-cache py3-pynacl ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt  && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.opnfv.txt -cupper-constraints.txt \
-        -e git+https://opendev.org/openstack/requirements@$OPENSTACK_TAG#egg=openstack_requirements && \
+    git init /src/requirements && \
+    (cd /src/requirements && \
+        git fetch --tags https://review.opendev.org/openstack/requirements $OPENSTACK_TAG && \
+        git checkout FETCH_HEAD) && \
     git init /src/functest && \
     (cd /src/functest && \
         git fetch --tags https://gerrit.opnfv.org/gerrit/functest $BRANCH && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/functest && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.opnfv.txt -cupper-constraints.txt \
-        /src/functest && \
+    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d /src/requirements/upper-constraints.txt && apk add --no-cache py3-pynacl ;; esac && \
+    sed -i -E /#egg=functest/d /src/functest/upper-constraints.txt && \
+    sed -i -E /^tempest==+.*$/d /src/requirements/upper-constraints.txt && \
+    pip3 install --no-cache-dir --src /src -c/src/functest/upper-constraints.txt -c/src/requirements/upper-constraints.txt \
+        -e /src/requirements && \
+    update-requirements -s --source /src/requirements /src/functest && \
+    pip3 install --no-cache-dir --src /src -c/src/functest/upper-constraints.txt -c/src/requirements/upper-constraints.txt \
+        -e /src/functest && \
     (cd /src/rally && patch -p1 < /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch) && \
-    rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/functest \
+    rm -r /src/requirements/.git /src/functest/.git \
         /tmp/Switch-to-threading.Thread-for-Rally-tasks.patch && \
-    cp /usr/lib/python3.8/site-packages/functest/ci/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
-    cp /usr/lib/python3.8/site-packages/functest/ci/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
+    cp /src/functest/functest/ci/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
+    cp /src/functest/functest/ci/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/ && \
     bash -c "mkdir -p /var/lib/xtesting /home/opnfv" && \
     ln -s /var/lib/xtesting /home/opnfv/functest && \
     bash -c "mkdir -p /home/opnfv/functest{/conf,/data,/images,/results} /home/opnfv/repos/vnfs" && \
diff --git a/docker/healthcheck/Dockerfile b/docker/healthcheck/Dockerfile
index 62c8e6138..4d55e93f9 100644
--- a/docker/healthcheck/Dockerfile
+++ b/docker/healthcheck/Dockerfile
@@ -1,32 +1,24 @@
 FROM opnfv/functest-core
 
-ARG BRANCH=master
-ARG OPENSTACK_TAG=master
 ARG ODL_TAG=89b88a0a23561f0bda62338b394ec41655679b2d
 ARG TEMPEST_HORIZON_TAG=master
 
 COPY thirdparty-requirements.txt thirdparty-requirements.txt
 RUN apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev openssl-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
     git init /src/tempest-horizon && \
     (cd /src/tempest-horizon && \
         git fetch --tags https://opendev.org/openstack/tempest-horizon.git $TEMPEST_HORIZON_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/tempest-horizon/ && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    update-requirements -s --source /src/requirements /src/tempest-horizon/ && \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         /src/tempest-horizon -rthirdparty-requirements.txt && \
     git init /src/odl_test && \
     (cd /src/odl_test && \
         git fetch --tags https://git.opendaylight.org/gerrit/integration/test $ODL_TAG && \
         git checkout FETCH_HEAD) && \
-    rm -r /src/odl_test/.git /src/tempest-horizon/ thirdparty-requirements.txt \
-        upper-constraints.txt upper-constraints.opnfv.txt && \
+    rm -r /src/odl_test/.git /src/tempest-horizon/ thirdparty-requirements.txt && \
     apk del .build-deps
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke-cntt/Dockerfile b/docker/smoke-cntt/Dockerfile
index c6b097093..4585edb34 100644
--- a/docker/smoke-cntt/Dockerfile
+++ b/docker/smoke-cntt/Dockerfile
@@ -1,5 +1,5 @@
 FROM opnfv/functest-smoke
 
 COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
-COPY tempest_conf.yaml /usr/lib/python3.8/site-packages/functest/opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml
+COPY tempest_conf.yaml /src/functest/functest/opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile
index 84e431d2d..927d78255 100644
--- a/docker/smoke/Dockerfile
+++ b/docker/smoke/Dockerfile
@@ -1,7 +1,5 @@
 FROM opnfv/functest-core
 
-ARG BRANCH=master
-ARG OPENSTACK_TAG=master
 ARG PATROLE_TAG=master
 ARG NEUTRON_TEMPEST_TAG=master
 ARG CINDER_TEMPEST_TAG=master
@@ -17,68 +15,62 @@ RUN apk --no-cache add --update libxml2 libxslt && \
     apk --no-cache add --virtual .build-deps --update \
         python3-dev build-base linux-headers libffi-dev \
         openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
-    case $(uname -m) in aarch*|arm*) CFLAGS="-O0" pip3 install --no-cache-dir -cupper-constraints.txt -cupper-constraints.opnfv.txt lxml ;; esac && \
     git init /src/patrole && \
     (cd /src/patrole && \
         git fetch --tags https://opendev.org/openstack/patrole.git $PATROLE_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/patrole/ && \
+    update-requirements -s --source /src/requirements /src/patrole/ && \
     git init /src/neutron-tempest-plugin && \
     (cd /src/neutron-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/neutron-tempest-plugin.git $NEUTRON_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/neutron-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/neutron-tempest-plugin && \
     git init /src/cinder-tempest-plugin && \
     (cd /src/cinder-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/cinder-tempest-plugin.git $CINDER_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/cinder-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/cinder-tempest-plugin && \
     git init /src/keystone-tempest-plugin && \
     (cd /src/keystone-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/keystone-tempest-plugin.git $KEYSTONE_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/keystone-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/keystone-tempest-plugin && \
     git init /src/barbican-tempest-plugin && \
     (cd /src/barbican-tempest-plugin && \
         git fetch --tags https://opendev.org/openstack/barbican-tempest-plugin.git $BARBICAN_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/barbican-tempest-plugin/ && \
+    update-requirements -s --source /src/requirements /src/barbican-tempest-plugin/ && \
     git init /src/octavia-tempest-plugin && \
     (cd /src/octavia-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/octavia-tempest-plugin.git $OCTAVIA_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/octavia-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/octavia-tempest-plugin && \
     git init /src/heat-tempest-plugin && \
     (cd /src/heat-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/heat-tempest-plugin.git $HEAT_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/heat-tempest-plugin && \
+    update-requirements -s --source /src/requirements /src/heat-tempest-plugin && \
     git init /src/telemetry-tempest-plugin && \
     (cd /src/telemetry-tempest-plugin && \
         git fetch --tags https://git.openstack.org/openstack/telemetry-tempest-plugin.git $TELEMETRY_TEMPEST_TAG && \
         git checkout FETCH_HEAD) && \
-    update-requirements -s --source /src/openstack-requirements /src/telemetry-tempest-plugin && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    update-requirements -s --source /src/requirements /src/telemetry-tempest-plugin && \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
         /src/cinder-tempest-plugin /src/keystone-tempest-plugin \
         /src/octavia-tempest-plugin /src/heat-tempest-plugin /src/telemetry-tempest-plugin && \
     mkdir -p /home/opnfv/functest/data/refstack && \
     mkdir -p /etc/neutron /etc/glance && \
     virtualenv --no-pip --no-setuptools --no-wheel oslo && . oslo/bin/activate && \
-    pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+    pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+        -c/src/functest/upper-constraints.txt \
         oslo.policy -e git+https://opendev.org/openstack/neutron.git@$NEUTRON_TAG#egg=neutron \
         -e git+https://opendev.org/openstack/glance.git@$GLANCE_TAG#egg=glance && \
     oslopolicy-sample-generator  --format json --output-file /etc/neutron/policy.json --namespace neutron && \
     oslopolicy-sample-generator  --format json --output-file /etc/glance/policy.json --namespace glance && \
     deactivate && \
-    rm -r oslo upper-constraints.txt upper-constraints.opnfv.txt \
-        /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
+    rm -r oslo /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
         /src/cinder-tempest-plugin /src/keystone-tempest-plugin \
         /src/octavia-tempest-plugin /src/heat-tempest-plugin \
         /src/telemetry-tempest-plugin /src/neutron /src/glance && \
diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile
index 3e477c05c..88f37484e 100644
--- a/docker/vnf/Dockerfile
+++ b/docker/vnf/Dockerfile
@@ -1,7 +1,5 @@
 FROM opnfv/functest-core
 
-ARG BRANCH=master
-ARG OPENSTACK_TAG=master
 ARG VIMS_TEST_TAG=release-130
 ARG QUAFF_TAG=59213d6d8ee29433552bb75f505cdc96b0b18909
 ARG CLOUDIFY_VIMS_TAG=gambia
@@ -23,13 +21,10 @@ RUN apk --no-cache add --update \
         procps libxslt libxml2 zlib libffi go musl-dev && \
     apk --no-cache add --virtual .build-deps --update \
         ruby-dev g++ make libxslt-dev libxml2-dev zlib-dev libffi-dev patch && \
-    wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
-    sed -i -E /^tempest==+.*$/d upper-constraints.txt && \
-    case $(uname -m) in aarch*|arm*) sed -i -E /^PyNaCl=/d upper-constraints.txt ;; esac && \
-    wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
-    sed -i -E /#egg=rally/d upper-constraints.opnfv.txt && \
-    git clone --depth 1 -b $VIMS_TEST_TAG https://github.com/Metaswitch/clearwater-live-test /src/vims-test && \
+    git init /src/vims-test && \
+    (cd /src/vims-test && \
+        git fetch --tags https://github.com/Metaswitch/clearwater-live-test $VIMS_TEST_TAG && \
+        git checkout FETCH_HEAD) && \
     sed -i s/unf_ext\ \(.*\)/unf_ext\ \(0.0.7.4\)/g /src/vims-test/Gemfile.lock && \
     git init /src/vims-test/quaff && \
     (cd /src/vims-test/quaff && \
@@ -64,7 +59,8 @@ RUN apk --no-cache add --update \
         (cd /src/epc-requirements/abot_charm && \
             git fetch --tags https://github.com/RebacaInc/abot_charm.git $ABOT_CHARM && \
             git checkout FETCH_HEAD) && \
-        pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+        pip3 install --no-cache-dir --src /src -c/src/requirements/upper-constraints.txt \
+            -c/src/functest/upper-constraints.txt \
             juju-wait==$JUJU_WAIT_TAG && \
         go get -d github.com/rogpeppe/godeps && \
         (cd $GOPATH/src/github.com/rogpeppe/godeps && git checkout $GODEPS_TAG && go install -v github.com/rogpeppe/godeps) && \
@@ -74,7 +70,7 @@ RUN apk --no-cache add --update \
         rm -r $GOPATH/src/ $GOPATH/pkg /src/epc-requirements/abot_charm/.git /root/.cache/go-build;; \
     esac && \
     (cd /src/vims-test && bundle config build.nokogiri --use-system-libraries && bundle install --system && bundle update rest-client) && \
-    rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \
+    rm -r /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \
         /src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \
         /tmp/clearwater-heat-singlenet-deps.patch && \
     apk del .build-deps