Update to Python3

Functest containers leverage on Python3 instead of python2.
https://mail.python.org/pipermail/python-dev/2018-March/152348.html

It also updates robotframework librairies to latest release and Vmtp
to master ([1] is needed)

It patches cloudify rest client to support python3.

Vmtp is currently disabled because it currently supports python2 only.

[1] https://github.com/openstack/vmtp/commit/a5d062881d91bf4f547d92c6e289bea30feb5d6e#diff-b4ef698db8ca845e5845c4618278f29a

Change-Id: I39964a212ec2d4dbf53c9ea7c63f02e0b6a05b48
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

2 files changed, 241 insertions, 4 deletions

diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile
index fc9294d56..8395cbfb0 100644
--- a/docker/vnf/Dockerfile
+++ b/docker/vnf/Dockerfile
@@ -18,15 +18,18 @@ ENV GOBIN /src/epc-requirements/go/bin
 ENV PATH $GOBIN:$PATH
 
 COPY clearwater-heat-singlenet-deps.patch /tmp/clearwater-heat-singlenet-deps.patch
+COPY cloudify-rest-client-py3.patch /tmp/cloudify-rest-client-py3.patch
 RUN apk --no-cache add --update \
         ruby ruby-bundler ruby-irb ruby-rdoc dnsmasq \
-        procps libxslt libxml2 zlib libffi python3 go musl-dev && \
+        procps libxslt libxml2 zlib libffi go musl-dev && \
     apk --no-cache add --virtual .build-deps --update \
         ruby-dev g++ make libxslt-dev libxml2-dev zlib-dev libffi-dev g++ make && \
     wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \
     sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/opendev.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
     wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
     sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
+    (cd /usr/lib/python3.6/site-packages/cloudify_rest_client && \
+        patch -p2 < /tmp/cloudify-rest-client-py3.patch) && \
     git clone --depth 1 -b $VIMS_TEST_TAG https://github.com/Metaswitch/clearwater-live-test /src/vims-test && \
     sed -i s/unf_ext\ \(.*\)/unf_ext\ \(0.0.7.4\)/g /src/vims-test/Gemfile.lock && \
     git init /src/vims-test/quaff && \
@@ -62,7 +65,7 @@ RUN apk --no-cache add --update \
         (cd /src/epc-requirements/abot_charm && \
             git fetch --tags https://github.com/RebacaInc/abot_charm.git $ABOT_CHARM && \
             git checkout FETCH_HEAD) && \
-        python3 -m pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
+        pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
             juju-wait==$JUJU_WAIT_TAG && \
         go get -d github.com/rogpeppe/godeps && \
         (cd $GOPATH/src/github.com/rogpeppe/godeps && git checkout $GODEPS_TAG && go install -v github.com/rogpeppe/godeps) && \
@@ -74,7 +77,7 @@ RUN apk --no-cache add --update \
     (cd /src/vims-test && bundle config build.nokogiri --use-system-libraries && bundle install --system && bundle update rest-client) && \
     rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \
         /src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \
-        /tmp/clearwater-heat-singlenet-deps.patch && \
+        /tmp/clearwater-heat-singlenet-deps.patch /tmp/cloudify-rest-client-py3.patch && \
     apk del .build-deps
-COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
+COPY testcases.yaml /usr/lib/python3.6/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/vnf/cloudify-rest-client-py3.patch b/docker/vnf/cloudify-rest-client-py3.patch
new file mode 100644
index 000000000..ef47e1aa3
--- /dev/null
+++ b/docker/vnf/cloudify-rest-client-py3.patch
@@ -0,0 +1,234 @@
+diff --git a/cloudify_rest_client/aria/mapi/mapi.py b/cloudify_rest_client/aria/mapi/mapi.py
+index 401c9de..5c4fa76 100644
+--- a/cloudify_rest_client/aria/mapi/mapi.py
++++ b/cloudify_rest_client/aria/mapi/mapi.py
+@@ -88,7 +88,7 @@ class RESTMAPI(api.ModelAPI):
+             wrapper = wrappers.DictWrapper
+             kw = dict(
+                     (key, self._wrap(value, '/'.join([attribute_path, key])))
+-                    for key, value in value.items()
++                    for key, value in list(value.items())
+                 )
+         elif isinstance(value, list):
+             wrapper = wrappers.ListWrapper
+diff --git a/cloudify_rest_client/aria/mapi/wrappers.py b/cloudify_rest_client/aria/mapi/wrappers.py
+index bdee6de..13af062 100644
+--- a/cloudify_rest_client/aria/mapi/wrappers.py
++++ b/cloudify_rest_client/aria/mapi/wrappers.py
+@@ -71,7 +71,7 @@ class DictWrapper(dict, WrapperBase):
+         )
+ 
+     def itervalues(self):
+-        return iter(self.values())
++        return iter(list(self.values()))
+ 
+     def items(self):
+         return [
+@@ -81,7 +81,7 @@ class DictWrapper(dict, WrapperBase):
+         ]
+ 
+     def iteritems(self):
+-        return iter(self.items())
++        return iter(list(self.items()))
+ 
+     def keys(self):
+         return [
+@@ -91,10 +91,10 @@ class DictWrapper(dict, WrapperBase):
+         ]
+ 
+     def iterkeys(self):
+-        return iter(self.keys())
++        return iter(list(self.keys()))
+ 
+     def __iter__(self):
+-        return self.iterkeys()
++        return iter(self.keys())
+ 
+ 
+ class ListWrapper(list, WrapperBase):
+diff --git a/cloudify_rest_client/aria/service_templates.py b/cloudify_rest_client/aria/service_templates.py
+index 05b86df..d49a91f 100644
+--- a/cloudify_rest_client/aria/service_templates.py
++++ b/cloudify_rest_client/aria/service_templates.py
+@@ -14,8 +14,8 @@
+ #    * limitations under the License.
+ 
+ import os
+-import urllib
+-import urlparse
++import urllib.request, urllib.parse, urllib.error
++import urllib.parse
+ from functools import partial
+ 
+ from .. import bytes_stream_utils
+@@ -41,14 +41,14 @@ class ServiceTemplateClient(BlueprintsClient):
+ 
+         if application_file_name is not None:
+             query_params['application_file_name'] = \
+-                urllib.quote(application_file_name)
++                urllib.parse.quote(application_file_name)
+ 
+         uri = '/{self._uri_prefix}/{id}'.format(self=self,
+                                                 id=service_template_id)
+ 
+         # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the
+         # drive letter and therefore the 2nd condition is present
+-        if urlparse.urlparse(archive_location).scheme and \
++        if urllib.parse.urlparse(archive_location).scheme and \
+                 not os.path.exists(archive_location):
+             # archive location is URL
+             query_params['service_template_csar_url'] = archive_location
+diff --git a/cloudify_rest_client/blueprints.py b/cloudify_rest_client/blueprints.py
+index 5d19325..5f533cc 100644
+--- a/cloudify_rest_client/blueprints.py
++++ b/cloudify_rest_client/blueprints.py
+@@ -16,8 +16,8 @@
+ import os
+ import tempfile
+ import shutil
+-import urllib
+-import urlparse
++import urllib.request, urllib.parse, urllib.error
++import urllib.parse
+ import contextlib
+ 
+ from cloudify_rest_client import utils
+@@ -95,13 +95,13 @@ class BlueprintsClient(object):
+         query_params = {'visibility': visibility}
+         if application_file_name is not None:
+             query_params['application_file_name'] = \
+-                urllib.quote(application_file_name)
++                urllib.parse.quote(application_file_name)
+ 
+         uri = '/{self._uri_prefix}/{id}'.format(self=self, id=blueprint_id)
+ 
+         # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the
+         # drive letter and therefore the 2nd condition is present
+-        if urlparse.urlparse(archive_location).scheme and \
++        if urllib.parse.urlparse(archive_location).scheme and \
+                 not os.path.exists(archive_location):
+             # archive location is URL
+             query_params['blueprint_archive_url'] = archive_location
+diff --git a/cloudify_rest_client/client.py b/cloudify_rest_client/client.py
+index 84641d2..0528e66 100644
+--- a/cloudify_rest_client/client.py
++++ b/cloudify_rest_client/client.py
+@@ -144,13 +144,13 @@ class HTTPClient(object):
+                                    verify=verify,
+                                    timeout=timeout or self.default_timeout_sec)
+         if self.logger.isEnabledFor(logging.DEBUG):
+-            for hdr, hdr_content in response.request.headers.iteritems():
++            for hdr, hdr_content in response.request.headers.items():
+                 self.logger.debug('request header:  %s: %s'
+                                   % (hdr, hdr_content))
+             self.logger.debug('reply:  "%s %s" %s'
+                               % (response.status_code,
+                                  response.reason, response.content))
+-            for hdr, hdr_content in response.headers.iteritems():
++            for hdr, hdr_content in response.headers.items():
+                 self.logger.debug('response header:  %s: %s'
+                                   % (hdr, hdr_content))
+ 
+@@ -209,7 +209,7 @@ class HTTPClient(object):
+         body = json.dumps(data) if is_dict_data else data
+         if self.logger.isEnabledFor(logging.DEBUG):
+             log_message = 'Sending request: {0} {1}'.format(
+-                requests_method.func_name.upper(),
++                requests_method.__name__.upper(),
+                 request_url)
+             if is_dict_data:
+                 log_message += '; body: {0}'.format(body)
+@@ -299,8 +299,8 @@ class HTTPClient(object):
+         if not username or not password:
+             return None
+         credentials = '{0}:{1}'.format(username, password)
+-        encoded_credentials = urlsafe_b64encode(credentials)
+-        return BASIC_AUTH_PREFIX + ' ' + encoded_credentials
++        encoded_credentials = urlsafe_b64encode(credentials.encode("utf-8"))
++        return BASIC_AUTH_PREFIX + ' ' + str(encoded_credentials, "utf-8")
+ 
+     def _set_header(self, key, value, log_value=True):
+         if not value:
+diff --git a/cloudify_rest_client/deployment_updates.py b/cloudify_rest_client/deployment_updates.py
+index 5010d86..d7db897 100644
+--- a/cloudify_rest_client/deployment_updates.py
++++ b/cloudify_rest_client/deployment_updates.py
+@@ -14,9 +14,9 @@
+ #    * limitations under the License.
+ import os
+ import json
+-import urllib
++import urllib.request, urllib.parse, urllib.error
+ import shutil
+-import urlparse
++import urllib.parse
+ import tempfile
+ from mimetypes import MimeTypes
+ 
+@@ -129,11 +129,11 @@ class DeploymentUpdatesClient(object):
+ 
+         if application_file_name:
+             params['application_file_name'] = \
+-                urllib.quote(application_file_name)
++                urllib.parse.quote(application_file_name)
+ 
+         # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the
+         # drive letter and therefore the 2nd condition is present
+-        if all([urlparse.urlparse(archive_path).scheme,
++        if all([urllib.parse.urlparse(archive_path).scheme,
+                 not os.path.exists(archive_path)]):
+             # archive location is URL
+             params['blueprint_archive_url'] = archive_path
+@@ -142,7 +142,7 @@ class DeploymentUpdatesClient(object):
+                 os.path.basename(archive_path),
+                 open(archive_path, 'rb'),
+                 # Guess the archive mime type
+-                mime_types.guess_type(urllib.pathname2url(archive_path)))
++                mime_types.guess_type(urllib.request.pathname2url(archive_path)))
+ 
+         return data_form, params
+ 
+diff --git a/cloudify_rest_client/plugins.py b/cloudify_rest_client/plugins.py
+index 76171b4..32e3e47 100644
+--- a/cloudify_rest_client/plugins.py
++++ b/cloudify_rest_client/plugins.py
+@@ -14,7 +14,7 @@
+ #    * limitations under the License.
+ 
+ import os
+-import urlparse
++import urllib.parse
+ import contextlib
+ 
+ from cloudify_rest_client import bytes_stream_utils
+@@ -222,7 +222,7 @@ class PluginsClient(object):
+         assert plugin_path
+         query_params = {'visibility': visibility}
+         timeout = self.api.default_timeout_sec
+-        if urlparse.urlparse(plugin_path).scheme and \
++        if urllib.parse.urlparse(plugin_path).scheme and \
+                 not os.path.exists(plugin_path):
+             query_params['plugin_archive_url'] = plugin_path
+             data = None
+diff --git a/cloudify_rest_client/snapshots.py b/cloudify_rest_client/snapshots.py
+index 851c3a3..0c40ceb 100644
+--- a/cloudify_rest_client/snapshots.py
++++ b/cloudify_rest_client/snapshots.py
+@@ -14,7 +14,7 @@
+ #    * limitations under the License.
+ 
+ import os
+-import urlparse
++import urllib.parse
+ import contextlib
+ 
+ from cloudify_rest_client import bytes_stream_utils
+@@ -192,7 +192,7 @@ class SnapshotsClient(object):
+         uri = '/snapshots/{0}/archive'.format(snapshot_id)
+         query_params = {}
+ 
+-        if urlparse.urlparse(snapshot_path).scheme and \
++        if urllib.parse.urlparse(snapshot_path).scheme and \
+                 not os.path.exists(snapshot_path):
+             query_params['snapshot_archive_url'] = snapshot_path
+             data = None