From f5aceb5447deed25234c5354b9dd9601115d9378 Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Mon, 18 Nov 2019 20:16:49 +0100 Subject: Forbid multipart upload if google storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Google Storage doesn't support S3 multipart uploads https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html https://cloud.google.com/storage/docs/composite-objects 5 * 1024 ** 5 is the multipart upload limit. https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html Change-Id: Iec3a5cd70ea6e912272ea231eecde6ee4f51ce4f Signed-off-by: Cédric Ollivier --- xtesting/core/testcase.py | 8 ++++++-- xtesting/tests/unit/core/test_testcase.py | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py index c3b8f61b..58de9f42 100644 --- a/xtesting/core/testcase.py +++ b/xtesting/core/testcase.py @@ -19,6 +19,7 @@ import re import sys import boto3 +from boto3.s3.transfer import TransferConfig import botocore import prettytable import requests @@ -282,6 +283,9 @@ class TestCase(): b3resource = boto3.resource( 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"]) dst_s3_url = os.environ["S3_DST_URL"] + multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[ + "S3_ENDPOINT_URL"] else 8 * 1024 * 1024 + config = TransferConfig(multipart_threshold=multipart_threshold) bucket_name = urllib.parse.urlparse(dst_s3_url).netloc try: b3resource.meta.client.head_bucket(Bucket=bucket_name) @@ -308,8 +312,7 @@ class TestCase(): "Publishing %s %s", abs_file, mime_type) # pylint: disable=no-member b3resource.Bucket(bucket_name).upload_file( - abs_file, - os.path.join(path, log_file), + abs_file, os.path.join(path, log_file), Config=config, ExtraArgs={'ContentType': mime_type[ 0] or 'application/octet-stream'}) link = os.path.join(dst_http_url, log_file) @@ -327,6 +330,7 @@ class TestCase(): os.path.join(path, os.path.relpath( os.path.join(root, pub_file), start=self.dir_results)), + Config=config, ExtraArgs={'ContentType': mime_type[ 0] or 'application/octet-stream'}) link = os.path.join(dst_http_url, os.path.relpath( diff --git a/xtesting/tests/unit/core/test_testcase.py b/xtesting/tests/unit/core/test_testcase.py index 5d34c34d..80e9d8cf 100644 --- a/xtesting/tests/unit/core/test_testcase.py +++ b/xtesting/tests/unit/core/test_testcase.py @@ -400,15 +400,18 @@ class TestCaseTesting(unittest.TestCase): mock.call().Bucket().upload_file( '/var/lib/xtesting/results/xtesting.log', 'prefix/xtesting.log', + Config=mock.ANY, ExtraArgs={'ContentType': 'application/octet-stream'}), mock.call().Bucket('xtesting'), mock.call().Bucket().upload_file( '/var/lib/xtesting/results/xtesting.debug.log', 'prefix/xtesting.debug.log', + Config=mock.ANY, ExtraArgs={'ContentType': 'application/octet-stream'}), mock.call().Bucket('xtesting'), mock.call().Bucket().upload_file( '/var/lib/xtesting/results/bar', 'prefix/bar', + Config=mock.ANY, ExtraArgs={'ContentType': 'application/octet-stream'})] self.assertEqual(args[1].mock_calls, expected) @@ -429,15 +432,18 @@ class TestCaseTesting(unittest.TestCase): mock.call().Bucket().upload_file( '/var/lib/xtesting/results/xtesting.log', 'prefix/xtesting.log', + Config=mock.ANY, ExtraArgs={'ContentType': 'text/plain'}), mock.call().Bucket('xtesting'), mock.call().Bucket().upload_file( '/var/lib/xtesting/results/xtesting.debug.log', 'prefix/xtesting.debug.log', + Config=mock.ANY, ExtraArgs={'ContentType': 'text/plain'}), mock.call().Bucket('xtesting'), mock.call().Bucket().upload_file( '/var/lib/xtesting/results/bar', 'prefix/bar', + Config=mock.ANY, ExtraArgs={'ContentType': 'text/plain'})] self.assertEqual(args[1].mock_calls, expected) -- cgit 1.2.3-korg