From 45baa7718c820bc712f24afa4bfa7d8d2ec2b4eb Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Thu, 7 Nov 2019 00:29:38 +0100 Subject: Create Bucket if it doesn't exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Xtesting Ansible Role no longer creates bucket after starting Minio [1]. [1] https://github.com/collivier/ansible-role-xtesting/commit/8c62525bb0d0b5f8dcc2c4436711d5d075de8811 Change-Id: I1ad473ce55f9c00df8e558fdc67f571ee0a1875c Signed-off-by: Cédric Ollivier (cherry picked from commit 7ddcfae7ff8a20fac21f5ba2b923e8e5cbce19e5) --- xtesting/core/testcase.py | 22 +++++++++++++++---- xtesting/tests/unit/core/test_testcase.py | 35 ++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py index 2db0f5c3..386c9d86 100644 --- a/xtesting/core/testcase.py +++ b/xtesting/core/testcase.py @@ -15,11 +15,12 @@ import json import logging import os import re -import requests +import sys import boto3 import botocore import prettytable +import requests import six from six.moves import urllib @@ -245,7 +246,7 @@ class TestCase(object): return TestCase.EX_PUSH_TO_DB_ERROR return TestCase.EX_OK - def publish_artifacts(self): + def publish_artifacts(self): # pylint: disable=too-many-locals """Push the artifacts to the S3 repository. It allows publishing the artifacts. @@ -272,13 +273,26 @@ class TestCase(object): b3resource = boto3.resource( 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"]) dst_s3_url = os.environ["S3_DST_URL"] - bucket = urllib.parse.urlparse(dst_s3_url).netloc + bucket_name = urllib.parse.urlparse(dst_s3_url).netloc + try: + b3resource.meta.client.head_bucket(Bucket=bucket_name) + except botocore.exceptions.ClientError as exc: + error_code = exc.response['Error']['Code'] + if error_code == '404': + # pylint: disable=no-member + b3resource.create_bucket(Bucket=bucket_name) + else: + typ, value, traceback = sys.exc_info() + six.reraise(typ, value, traceback) + except Exception: # pylint: disable=broad-except + typ, value, traceback = sys.exc_info() + six.reraise(typ, value, traceback) path = urllib.parse.urlparse(dst_s3_url).path.strip("/") output_str = "\n" for root, _, files in os.walk(self.dir_results): for pub_file in files: # pylint: disable=no-member - b3resource.Bucket(bucket).upload_file( + b3resource.Bucket(bucket_name).upload_file( os.path.join(root, pub_file), os.path.join(path, os.path.relpath( os.path.join(root, pub_file), diff --git a/xtesting/tests/unit/core/test_testcase.py b/xtesting/tests/unit/core/test_testcase.py index eff64d5a..3dbbac2c 100644 --- a/xtesting/tests/unit/core/test_testcase.py +++ b/xtesting/tests/unit/core/test_testcase.py @@ -339,6 +339,25 @@ class TestCaseTesting(unittest.TestCase): args[0].assert_called_once_with( 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + @mock.patch('boto3.resource') + def test_publish_artifacts_exc4(self, *args): + args[0].return_value.meta.client.head_bucket.side_effect = Exception + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_PUBLISH_ARTIFACTS_ERROR) + args[0].assert_called_once_with( + 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + + @mock.patch('boto3.resource') + def test_publish_artifacts_exc5(self, *args): + error_response = {'Error': {'Code': '403'}} + mock_head_bucket = args[0].return_value.meta.client.head_bucket + mock_head_bucket.side_effect = botocore.exceptions.ClientError( + error_response, 'Foo') + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_PUBLISH_ARTIFACTS_ERROR) + args[0].assert_called_once_with( + 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + @mock.patch('boto3.resource') @mock.patch('os.walk', return_value=[]) def test_publish_artifacts1(self, *args): @@ -348,16 +367,30 @@ class TestCaseTesting(unittest.TestCase): args[1].assert_called_once_with( 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + @mock.patch('boto3.resource') + @mock.patch('os.walk', return_value=[]) + def test_publish_artifacts2(self, *args): + error_response = {'Error': {'Code': '404'}} + mock_head_bucket = args[1].return_value.meta.client.head_bucket + mock_head_bucket.side_effect = botocore.exceptions.ClientError( + error_response, 'NoSuchBucket') + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_OK) + args[0].assert_called_once_with(self.test.dir_results) + args[1].assert_called_once_with( + 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + @mock.patch('boto3.resource') @mock.patch('os.walk', return_value=[ (testcase.TestCase.dir_results, ('',), ('bar',))]) - def test_publish_artifacts2(self, *args): + def test_publish_artifacts3(self, *args): self.assertEqual(self.test.publish_artifacts(), testcase.TestCase.EX_OK) args[0].assert_called_once_with(self.test.dir_results) expected = [ mock.call('s3', endpoint_url=os.environ['S3_ENDPOINT_URL']), + mock.call().meta.client.head_bucket(Bucket='xtesting'), mock.call().Bucket('xtesting'), mock.call().Bucket().upload_file( '/var/lib/xtesting/results/bar', 'prefix/bar')] -- cgit 1.2.3-korg