From 2a0702ea914a574ffa7c6d6f83a5c606e24ebd0a Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Tue, 7 May 2019 11:02:43 +0200 Subject: Update to Python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Functest containers leverage on Python3 instead of python2. https://mail.python.org/pipermail/python-dev/2018-March/152348.html It also updates robotframework librairies to latest release and Vmtp to master ([1] is needed) It patches cloudify rest client to support python3. Vmtp is currently disabled because it currently supports python2 only. [1] https://github.com/openstack/vmtp/commit/a5d062881d91bf4f547d92c6e289bea30feb5d6e#diff-b4ef698db8ca845e5845c4618278f29a Change-Id: I39964a212ec2d4dbf53c9ea7c63f02e0b6a05b48 Signed-off-by: Cédric Ollivier --- docker/vnf/Dockerfile | 11 +- docker/vnf/cloudify-rest-client-py3.patch | 234 ++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 docker/vnf/cloudify-rest-client-py3.patch (limited to 'docker/vnf') diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile index fc9294d56..8395cbfb0 100644 --- a/docker/vnf/Dockerfile +++ b/docker/vnf/Dockerfile @@ -18,15 +18,18 @@ ENV GOBIN /src/epc-requirements/go/bin ENV PATH $GOBIN:$PATH COPY clearwater-heat-singlenet-deps.patch /tmp/clearwater-heat-singlenet-deps.patch +COPY cloudify-rest-client-py3.patch /tmp/cloudify-rest-client-py3.patch RUN apk --no-cache add --update \ ruby ruby-bundler ruby-irb ruby-rdoc dnsmasq \ - procps libxslt libxml2 zlib libffi python3 go musl-dev && \ + procps libxslt libxml2 zlib libffi go musl-dev && \ apk --no-cache add --virtual .build-deps --update \ ruby-dev g++ make libxslt-dev libxml2-dev zlib-dev libffi-dev g++ make && \ wget -q -O- https://opendev.org/openstack/requirements/raw/branch/$OPENSTACK_TAG/upper-constraints.txt > upper-constraints.txt && \ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/opendev.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ + (cd /usr/lib/python3.6/site-packages/cloudify_rest_client && \ + patch -p2 < /tmp/cloudify-rest-client-py3.patch) && \ git clone --depth 1 -b $VIMS_TEST_TAG https://github.com/Metaswitch/clearwater-live-test /src/vims-test && \ sed -i s/unf_ext\ \(.*\)/unf_ext\ \(0.0.7.4\)/g /src/vims-test/Gemfile.lock && \ git init /src/vims-test/quaff && \ @@ -62,7 +65,7 @@ RUN apk --no-cache add --update \ (cd /src/epc-requirements/abot_charm && \ git fetch --tags https://github.com/RebacaInc/abot_charm.git $ABOT_CHARM && \ git checkout FETCH_HEAD) && \ - python3 -m pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ + pip3 install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ juju-wait==$JUJU_WAIT_TAG && \ go get -d github.com/rogpeppe/godeps && \ (cd $GOPATH/src/github.com/rogpeppe/godeps && git checkout $GODEPS_TAG && go install -v github.com/rogpeppe/godeps) && \ @@ -74,7 +77,7 @@ RUN apk --no-cache add --update \ (cd /src/vims-test && bundle config build.nokogiri --use-system-libraries && bundle install --system && bundle update rest-client) && \ rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \ /src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \ - /tmp/clearwater-heat-singlenet-deps.patch && \ + /tmp/clearwater-heat-singlenet-deps.patch /tmp/cloudify-rest-client-py3.patch && \ apk del .build-deps -COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml +COPY testcases.yaml /usr/lib/python3.6/site-packages/xtesting/ci/testcases.yaml CMD ["run_tests", "-t", "all"] diff --git a/docker/vnf/cloudify-rest-client-py3.patch b/docker/vnf/cloudify-rest-client-py3.patch new file mode 100644 index 000000000..ef47e1aa3 --- /dev/null +++ b/docker/vnf/cloudify-rest-client-py3.patch @@ -0,0 +1,234 @@ +diff --git a/cloudify_rest_client/aria/mapi/mapi.py b/cloudify_rest_client/aria/mapi/mapi.py +index 401c9de..5c4fa76 100644 +--- a/cloudify_rest_client/aria/mapi/mapi.py ++++ b/cloudify_rest_client/aria/mapi/mapi.py +@@ -88,7 +88,7 @@ class RESTMAPI(api.ModelAPI): + wrapper = wrappers.DictWrapper + kw = dict( + (key, self._wrap(value, '/'.join([attribute_path, key]))) +- for key, value in value.items() ++ for key, value in list(value.items()) + ) + elif isinstance(value, list): + wrapper = wrappers.ListWrapper +diff --git a/cloudify_rest_client/aria/mapi/wrappers.py b/cloudify_rest_client/aria/mapi/wrappers.py +index bdee6de..13af062 100644 +--- a/cloudify_rest_client/aria/mapi/wrappers.py ++++ b/cloudify_rest_client/aria/mapi/wrappers.py +@@ -71,7 +71,7 @@ class DictWrapper(dict, WrapperBase): + ) + + def itervalues(self): +- return iter(self.values()) ++ return iter(list(self.values())) + + def items(self): + return [ +@@ -81,7 +81,7 @@ class DictWrapper(dict, WrapperBase): + ] + + def iteritems(self): +- return iter(self.items()) ++ return iter(list(self.items())) + + def keys(self): + return [ +@@ -91,10 +91,10 @@ class DictWrapper(dict, WrapperBase): + ] + + def iterkeys(self): +- return iter(self.keys()) ++ return iter(list(self.keys())) + + def __iter__(self): +- return self.iterkeys() ++ return iter(self.keys()) + + + class ListWrapper(list, WrapperBase): +diff --git a/cloudify_rest_client/aria/service_templates.py b/cloudify_rest_client/aria/service_templates.py +index 05b86df..d49a91f 100644 +--- a/cloudify_rest_client/aria/service_templates.py ++++ b/cloudify_rest_client/aria/service_templates.py +@@ -14,8 +14,8 @@ + # * limitations under the License. + + import os +-import urllib +-import urlparse ++import urllib.request, urllib.parse, urllib.error ++import urllib.parse + from functools import partial + + from .. import bytes_stream_utils +@@ -41,14 +41,14 @@ class ServiceTemplateClient(BlueprintsClient): + + if application_file_name is not None: + query_params['application_file_name'] = \ +- urllib.quote(application_file_name) ++ urllib.parse.quote(application_file_name) + + uri = '/{self._uri_prefix}/{id}'.format(self=self, + id=service_template_id) + + # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the + # drive letter and therefore the 2nd condition is present +- if urlparse.urlparse(archive_location).scheme and \ ++ if urllib.parse.urlparse(archive_location).scheme and \ + not os.path.exists(archive_location): + # archive location is URL + query_params['service_template_csar_url'] = archive_location +diff --git a/cloudify_rest_client/blueprints.py b/cloudify_rest_client/blueprints.py +index 5d19325..5f533cc 100644 +--- a/cloudify_rest_client/blueprints.py ++++ b/cloudify_rest_client/blueprints.py +@@ -16,8 +16,8 @@ + import os + import tempfile + import shutil +-import urllib +-import urlparse ++import urllib.request, urllib.parse, urllib.error ++import urllib.parse + import contextlib + + from cloudify_rest_client import utils +@@ -95,13 +95,13 @@ class BlueprintsClient(object): + query_params = {'visibility': visibility} + if application_file_name is not None: + query_params['application_file_name'] = \ +- urllib.quote(application_file_name) ++ urllib.parse.quote(application_file_name) + + uri = '/{self._uri_prefix}/{id}'.format(self=self, id=blueprint_id) + + # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the + # drive letter and therefore the 2nd condition is present +- if urlparse.urlparse(archive_location).scheme and \ ++ if urllib.parse.urlparse(archive_location).scheme and \ + not os.path.exists(archive_location): + # archive location is URL + query_params['blueprint_archive_url'] = archive_location +diff --git a/cloudify_rest_client/client.py b/cloudify_rest_client/client.py +index 84641d2..0528e66 100644 +--- a/cloudify_rest_client/client.py ++++ b/cloudify_rest_client/client.py +@@ -144,13 +144,13 @@ class HTTPClient(object): + verify=verify, + timeout=timeout or self.default_timeout_sec) + if self.logger.isEnabledFor(logging.DEBUG): +- for hdr, hdr_content in response.request.headers.iteritems(): ++ for hdr, hdr_content in response.request.headers.items(): + self.logger.debug('request header: %s: %s' + % (hdr, hdr_content)) + self.logger.debug('reply: "%s %s" %s' + % (response.status_code, + response.reason, response.content)) +- for hdr, hdr_content in response.headers.iteritems(): ++ for hdr, hdr_content in response.headers.items(): + self.logger.debug('response header: %s: %s' + % (hdr, hdr_content)) + +@@ -209,7 +209,7 @@ class HTTPClient(object): + body = json.dumps(data) if is_dict_data else data + if self.logger.isEnabledFor(logging.DEBUG): + log_message = 'Sending request: {0} {1}'.format( +- requests_method.func_name.upper(), ++ requests_method.__name__.upper(), + request_url) + if is_dict_data: + log_message += '; body: {0}'.format(body) +@@ -299,8 +299,8 @@ class HTTPClient(object): + if not username or not password: + return None + credentials = '{0}:{1}'.format(username, password) +- encoded_credentials = urlsafe_b64encode(credentials) +- return BASIC_AUTH_PREFIX + ' ' + encoded_credentials ++ encoded_credentials = urlsafe_b64encode(credentials.encode("utf-8")) ++ return BASIC_AUTH_PREFIX + ' ' + str(encoded_credentials, "utf-8") + + def _set_header(self, key, value, log_value=True): + if not value: +diff --git a/cloudify_rest_client/deployment_updates.py b/cloudify_rest_client/deployment_updates.py +index 5010d86..d7db897 100644 +--- a/cloudify_rest_client/deployment_updates.py ++++ b/cloudify_rest_client/deployment_updates.py +@@ -14,9 +14,9 @@ + # * limitations under the License. + import os + import json +-import urllib ++import urllib.request, urllib.parse, urllib.error + import shutil +-import urlparse ++import urllib.parse + import tempfile + from mimetypes import MimeTypes + +@@ -129,11 +129,11 @@ class DeploymentUpdatesClient(object): + + if application_file_name: + params['application_file_name'] = \ +- urllib.quote(application_file_name) ++ urllib.parse.quote(application_file_name) + + # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the + # drive letter and therefore the 2nd condition is present +- if all([urlparse.urlparse(archive_path).scheme, ++ if all([urllib.parse.urlparse(archive_path).scheme, + not os.path.exists(archive_path)]): + # archive location is URL + params['blueprint_archive_url'] = archive_path +@@ -142,7 +142,7 @@ class DeploymentUpdatesClient(object): + os.path.basename(archive_path), + open(archive_path, 'rb'), + # Guess the archive mime type +- mime_types.guess_type(urllib.pathname2url(archive_path))) ++ mime_types.guess_type(urllib.request.pathname2url(archive_path))) + + return data_form, params + +diff --git a/cloudify_rest_client/plugins.py b/cloudify_rest_client/plugins.py +index 76171b4..32e3e47 100644 +--- a/cloudify_rest_client/plugins.py ++++ b/cloudify_rest_client/plugins.py +@@ -14,7 +14,7 @@ + # * limitations under the License. + + import os +-import urlparse ++import urllib.parse + import contextlib + + from cloudify_rest_client import bytes_stream_utils +@@ -222,7 +222,7 @@ class PluginsClient(object): + assert plugin_path + query_params = {'visibility': visibility} + timeout = self.api.default_timeout_sec +- if urlparse.urlparse(plugin_path).scheme and \ ++ if urllib.parse.urlparse(plugin_path).scheme and \ + not os.path.exists(plugin_path): + query_params['plugin_archive_url'] = plugin_path + data = None +diff --git a/cloudify_rest_client/snapshots.py b/cloudify_rest_client/snapshots.py +index 851c3a3..0c40ceb 100644 +--- a/cloudify_rest_client/snapshots.py ++++ b/cloudify_rest_client/snapshots.py +@@ -14,7 +14,7 @@ + # * limitations under the License. + + import os +-import urlparse ++import urllib.parse + import contextlib + + from cloudify_rest_client import bytes_stream_utils +@@ -192,7 +192,7 @@ class SnapshotsClient(object): + uri = '/snapshots/{0}/archive'.format(snapshot_id) + query_params = {} + +- if urlparse.urlparse(snapshot_path).scheme and \ ++ if urllib.parse.urlparse(snapshot_path).scheme and \ + not os.path.exists(snapshot_path): + query_params['snapshot_archive_url'] = snapshot_path + data = None -- cgit 1.2.3-korg