From b407dc6f6db37925ee3a007690982efeedfde65a Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Sat, 2 Nov 2019 12:18:22 +0100 Subject: Publish artifacts to S3 repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It simplifies Jenkins or Gitlab jobs by automatically publishing all artifacts via the framework. It leverages on Amazon Web Services (AWS) SDK [1] which supports the current cases (OPNFV, Xtesting Ansible role [2], etc.). [1] https://boto3.amazonaws.com/v1/documentation/api/latest/index.html?id=docs_gateway [2] https://github.com/collivier/ansible-role-xtesting Change-Id: I66e380c4da29fb0f973472a2c59ae0ea3c44fcfd Signed-off-by: Cédric Ollivier (cherry picked from commit d012f3ac3ec4aa2730532be095956867d797aefb) --- requirements.txt | 1 + upper-constraints.txt | 1 + xtesting/ci/run_tests.py | 10 ++++- xtesting/core/feature.py | 1 - xtesting/core/robotframework.py | 1 - xtesting/core/testcase.py | 66 +++++++++++++++++++++++++++++++ xtesting/core/unit.py | 1 - xtesting/tests/unit/core/test_testcase.py | 52 ++++++++++++++++++++++++ 8 files changed, 129 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6be92273..6496a3da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ six # MIT python-subunit # Apache-2.0/BSD os-testr # Apache-2.0 junitxml +boto3 # Apache-2.0 diff --git a/upper-constraints.txt b/upper-constraints.txt index 1e03562c..96928557 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -2,3 +2,4 @@ robotframework===3.0.2 bandit===1.1.0 pylint===1.9.5;python_version=='2.7' pylint===2.3.1;python_version=='3.6' +boto3===1.9.108 diff --git a/xtesting/ci/run_tests.py b/xtesting/ci/run_tests.py index 7ec4dfce..e3a5986f 100644 --- a/xtesting/ci/run_tests.py +++ b/xtesting/ci/run_tests.py @@ -65,6 +65,9 @@ class RunTestsParser(): self.parser.add_argument("-r", "--report", help="Push results to " "database (default=false).", action="store_true") + self.parser.add_argument("-p", "--push", help="Push artifacts to " + "S3 repository (default=false).", + action="store_true") def parse_args(self, argv=None): """Parse arguments. @@ -85,6 +88,7 @@ class Runner(): self.overall_result = Result.EX_OK self.clean_flag = True self.report_flag = False + self.push_flag = False self.tiers = tier_builder.TierBuilder( pkg_resources.resource_filename('xtesting', 'ci/testcases.yaml')) @@ -174,6 +178,8 @@ class Runner(): LOGGER.info("Test result:\n\n%s\n", test_case) if self.clean_flag: test_case.clean() + if self.push_flag: + test_case.publish_artifacts() except ImportError: LOGGER.exception("Cannot import module %s", run_dict['module']) except AttributeError: @@ -226,12 +232,14 @@ class Runner(): for tier in tiers_to_run: self.run_tier(tier) - def main(self, **kwargs): + def main(self, **kwargs): # pylint: disable=too-many-branches """Entry point of class Runner""" if 'noclean' in kwargs: self.clean_flag = not kwargs['noclean'] if 'report' in kwargs: self.report_flag = kwargs['report'] + if 'push' in kwargs: + self.push_flag = kwargs['push'] try: LOGGER.info("Deployment description:\n\n%s\n", env.string()) self.source_envfile() diff --git a/xtesting/core/feature.py b/xtesting/core/feature.py index f28e720c..3b2a19f2 100644 --- a/xtesting/core/feature.py +++ b/xtesting/core/feature.py @@ -88,7 +88,6 @@ class BashFeature(Feature): def __init__(self, **kwargs): super(BashFeature, self).__init__(**kwargs) - self.res_dir = "/var/lib/xtesting/results/{}".format(self.case_name) self.result_file = "{}/{}.log".format(self.res_dir, self.case_name) def execute(self, **kwargs): diff --git a/xtesting/core/robotframework.py b/xtesting/core/robotframework.py index 2791b559..c2fec56a 100644 --- a/xtesting/core/robotframework.py +++ b/xtesting/core/robotframework.py @@ -58,7 +58,6 @@ class RobotFramework(testcase.TestCase): def __init__(self, **kwargs): super(RobotFramework, self).__init__(**kwargs) - self.res_dir = os.path.join(self.dir_results, self.case_name) self.xml_file = os.path.join(self.res_dir, 'output.xml') def parse_results(self): diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py index c89e4c88..785f6c85 100644 --- a/xtesting/core/testcase.py +++ b/xtesting/core/testcase.py @@ -17,8 +17,11 @@ import os import re import requests +import boto3 +import botocore import prettytable import six +from six.moves import urllib from xtesting.utils import decorators from xtesting.utils import env @@ -46,6 +49,10 @@ class TestCase(): EX_TESTCASE_SKIPPED = os.EX_SOFTWARE - 3 """requirements are unmet""" + EX_PUBLISH_ARTIFACTS_ERROR = os.EX_SOFTWARE - 4 + """publish_artifacts() failed""" + + dir_results = "/var/lib/xtesting/results" _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*" _headers = {'Content-Type': 'application/json'} __logger = logging.getLogger(__name__) @@ -59,6 +66,7 @@ class TestCase(): self.start_time = 0 self.stop_time = 0 self.is_skipped = False + self.res_dir = "{}/{}".format(self.dir_results, self.case_name) def __str__(self): try: @@ -237,6 +245,64 @@ class TestCase(): return TestCase.EX_PUSH_TO_DB_ERROR return TestCase.EX_OK + def publish_artifacts(self): + """Push the artifacts to the S3 repository. + + It allows publishing the artifacts. + + It could be overriden if the common implementation is not + suitable. + + The credentials must be configured before publishing the artifacts: + + * fill ~/.aws/credentials or ~/.boto, + * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env. + + The next vars must be set in env: + + * S3_ENDPOINT_URL (http://127.0.0.1:9000), + * S3_DST_URL (s3://xtesting/prefix), + * HTTP_DST_URL (http://127.0.0.1/prefix). + + Returns: + TestCase.EX_OK if artifacts were published to repository. + TestCase.EX_PUBLISH_ARTIFACTS_ERROR otherwise. + """ + try: + b3resource = boto3.resource( + 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"]) + dst_s3_url = os.environ["S3_DST_URL"] + bucket = urllib.parse.urlparse(dst_s3_url).netloc + path = urllib.parse.urlparse(dst_s3_url).path.strip("/") + output_str = "\n" + for root, _, files in os.walk(self.dir_results): + for pub_file in files: + # pylint: disable=no-member + b3resource.Bucket(bucket).upload_file( + os.path.join(root, pub_file), + os.path.join(path, os.path.relpath( + os.path.join(root, pub_file), + start=self.dir_results))) + dst_http_url = os.environ["HTTP_DST_URL"] + output_str += "\n{}".format( + os.path.join(dst_http_url, os.path.relpath( + os.path.join(root, pub_file), + start=self.dir_results))) + self.__logger.info( + "All artifacts were successfully published: %s\n", output_str) + return TestCase.EX_OK + except KeyError as ex: + self.__logger.error("Please check env var: %s", str(ex)) + return TestCase.EX_PUBLISH_ARTIFACTS_ERROR + except botocore.exceptions.NoCredentialsError: + self.__logger.error( + "Please fill ~/.aws/credentials, ~/.boto or set " + "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env") + return TestCase.EX_PUBLISH_ARTIFACTS_ERROR + except Exception: # pylint: disable=broad-except + self.__logger.exception("Cannot publish the artifacts") + return TestCase.EX_PUBLISH_ARTIFACTS_ERROR + def clean(self): """Clean the resources. diff --git a/xtesting/core/unit.py b/xtesting/core/unit.py index 774411a4..877cd073 100644 --- a/xtesting/core/unit.py +++ b/xtesting/core/unit.py @@ -34,7 +34,6 @@ class Suite(testcase.TestCase): def __init__(self, **kwargs): super(Suite, self).__init__(**kwargs) - self.res_dir = "/var/lib/xtesting/results/{}".format(self.case_name) self.suite = None @classmethod diff --git a/xtesting/tests/unit/core/test_testcase.py b/xtesting/tests/unit/core/test_testcase.py index fc612973..eff64d5a 100644 --- a/xtesting/tests/unit/core/test_testcase.py +++ b/xtesting/tests/unit/core/test_testcase.py @@ -17,6 +17,7 @@ import logging import os import unittest +import botocore import mock import requests @@ -61,6 +62,9 @@ class TestCaseTesting(unittest.TestCase): os.environ['DEPLOY_SCENARIO'] = "scenario" os.environ['NODE_NAME'] = "node_name" os.environ['BUILD_TAG'] = "foo-daily-master-bar" + os.environ['S3_ENDPOINT_URL'] = "http://127.0.0.1:9000" + os.environ['S3_DST_URL'] = "s3://xtesting/prefix" + os.environ['HTTP_DST_URL'] = "http://127.0.0.1/prefix" def test_run_fake(self): self.assertEqual(self.test.run(), testcase.TestCase.EX_OK) @@ -311,6 +315,54 @@ class TestCaseTesting(unittest.TestCase): def test_clean(self): self.assertEqual(self.test.clean(), None) + def _test_publish_artifacts_nokw(self, key): + del os.environ[key] + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_PUBLISH_ARTIFACTS_ERROR) + + def test_publish_artifacts_exc1(self): + for key in ["S3_ENDPOINT_URL", "S3_DST_URL", "HTTP_DST_URL"]: + self._test_publish_artifacts_nokw(key) + + @mock.patch('boto3.resource', + side_effect=botocore.exceptions.NoCredentialsError) + def test_publish_artifacts_exc2(self, *args): + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_PUBLISH_ARTIFACTS_ERROR) + args[0].assert_called_once_with( + 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + + @mock.patch('boto3.resource', side_effect=Exception) + def test_publish_artifacts_exc3(self, *args): + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_PUBLISH_ARTIFACTS_ERROR) + args[0].assert_called_once_with( + 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + + @mock.patch('boto3.resource') + @mock.patch('os.walk', return_value=[]) + def test_publish_artifacts1(self, *args): + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_OK) + args[0].assert_called_once_with(self.test.dir_results) + args[1].assert_called_once_with( + 's3', endpoint_url=os.environ['S3_ENDPOINT_URL']) + + @mock.patch('boto3.resource') + @mock.patch('os.walk', + return_value=[ + (testcase.TestCase.dir_results, ('',), ('bar',))]) + def test_publish_artifacts2(self, *args): + self.assertEqual(self.test.publish_artifacts(), + testcase.TestCase.EX_OK) + args[0].assert_called_once_with(self.test.dir_results) + expected = [ + mock.call('s3', endpoint_url=os.environ['S3_ENDPOINT_URL']), + mock.call().Bucket('xtesting'), + mock.call().Bucket().upload_file( + '/var/lib/xtesting/results/bar', 'prefix/bar')] + self.assertEqual(args[1].mock_calls, expected) + if __name__ == "__main__": logging.disable(logging.CRITICAL) -- cgit 1.2.3-korg