From f5aceb5447deed25234c5354b9dd9601115d9378 Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Mon, 18 Nov 2019 20:16:49 +0100 Subject: Forbid multipart upload if google storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Google Storage doesn't support S3 multipart uploads https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html https://cloud.google.com/storage/docs/composite-objects 5 * 1024 ** 5 is the multipart upload limit. https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html Change-Id: Iec3a5cd70ea6e912272ea231eecde6ee4f51ce4f Signed-off-by: Cédric Ollivier --- xtesting/core/testcase.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'xtesting/core') diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py index c3b8f61b..58de9f42 100644 --- a/xtesting/core/testcase.py +++ b/xtesting/core/testcase.py @@ -19,6 +19,7 @@ import re import sys import boto3 +from boto3.s3.transfer import TransferConfig import botocore import prettytable import requests @@ -282,6 +283,9 @@ class TestCase(): b3resource = boto3.resource( 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"]) dst_s3_url = os.environ["S3_DST_URL"] + multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[ + "S3_ENDPOINT_URL"] else 8 * 1024 * 1024 + config = TransferConfig(multipart_threshold=multipart_threshold) bucket_name = urllib.parse.urlparse(dst_s3_url).netloc try: b3resource.meta.client.head_bucket(Bucket=bucket_name) @@ -308,8 +312,7 @@ class TestCase(): "Publishing %s %s", abs_file, mime_type) # pylint: disable=no-member b3resource.Bucket(bucket_name).upload_file( - abs_file, - os.path.join(path, log_file), + abs_file, os.path.join(path, log_file), Config=config, ExtraArgs={'ContentType': mime_type[ 0] or 'application/octet-stream'}) link = os.path.join(dst_http_url, log_file) @@ -327,6 +330,7 @@ class TestCase(): os.path.join(path, os.path.relpath( os.path.join(root, pub_file), start=self.dir_results)), + Config=config, ExtraArgs={'ContentType': mime_type[ 0] or 'application/octet-stream'}) link = os.path.join(dst_http_url, os.path.relpath( -- cgit 1.2.3-korg