From 5ab58cf461b7803e4c9c5c0983e8c397468d27a8 Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Thu, 3 Mar 2022 15:29:16 +0100 Subject: Search config files in tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It allows putting configurations in classical dirs. It fallbacks to the python package for backward compatibility. Change-Id: Ie33b9482fb197926c7d7d66ace815fa4ae01d02d Signed-off-by: Cédric Ollivier --- xtesting/ci/run_tests.py | 21 ++++++++++++--------- xtesting/ci/testcases.yaml | 6 +++--- xtesting/core/campaign.py | 18 +++++++++--------- xtesting/utils/config.py | 23 +++++++++++++++++++++++ xtesting/utils/constants.py | 23 +++++++++++++++++++++-- 5 files changed, 68 insertions(+), 23 deletions(-) create mode 100644 xtesting/utils/config.py (limited to 'xtesting') diff --git a/xtesting/ci/run_tests.py b/xtesting/ci/run_tests.py index 16e7ef16..0b94c28b 100644 --- a/xtesting/ci/run_tests.py +++ b/xtesting/ci/run_tests.py @@ -23,13 +23,13 @@ import sys import textwrap import enum -import pkg_resources import prettytable from stevedore import driver import yaml from xtesting.ci import tier_builder from xtesting.core import testcase +from xtesting.utils import config from xtesting.utils import constants from xtesting.utils import env @@ -88,8 +88,9 @@ class Runner(): self.clean_flag = True self.report_flag = False self.push_flag = False - self.tiers = tier_builder.TierBuilder( - pkg_resources.resource_filename('xtesting', 'ci/testcases.yaml')) + self.tiers = tier_builder.TierBuilder(config.get_xtesting_config( + constants.TESTCASE_DESCRIPTION, + constants.TESTCASE_DESCRIPTION_DEFAULT)) @staticmethod def source_envfile(rc_file=constants.ENV_FILE): @@ -114,8 +115,10 @@ class Runner(): @staticmethod def get_dict_by_test(testname): # pylint: disable=missing-docstring - with open(pkg_resources.resource_filename( - 'xtesting', 'ci/testcases.yaml'), encoding='utf-8') as tyaml: + with open(config.get_xtesting_config( + constants.TESTCASE_DESCRIPTION, + constants.TESTCASE_DESCRIPTION_DEFAULT), + encoding='utf-8') as tyaml: testcases_yaml = yaml.safe_load(tyaml) for dic_tier in testcases_yaml.get("tiers"): for dic_testcase in dic_tier['testcases']: @@ -313,11 +316,11 @@ def main(): print(f"Cannot create {constants.RESULTS_DIR}") return testcase.TestCase.EX_RUN_ERROR if env.get('DEBUG').lower() == 'true': - logging.config.fileConfig(pkg_resources.resource_filename( - 'xtesting', constants.DEBUG_INI_PATH)) + logging.config.fileConfig(config.get_xtesting_config( + 'logging.debug.ini', constants.DEBUG_INI_PATH_DEFAULT)) else: - logging.config.fileConfig(pkg_resources.resource_filename( - 'xtesting', constants.INI_PATH)) + logging.config.fileConfig(config.get_xtesting_config( + 'logging.ini', constants.INI_PATH_DEFAULT)) logging.captureWarnings(True) parser = RunTestsParser() args = parser.parse_args(sys.argv[1:]) diff --git a/xtesting/ci/testcases.yaml b/xtesting/ci/testcases.yaml index 16dd2632..a5f3f8ea 100644 --- a/xtesting/ci/testcases.yaml +++ b/xtesting/ci/testcases.yaml @@ -51,7 +51,7 @@ tiers: args: suites: - >- - /usr/lib/python3.8/site-packages/xtesting/samples/HelloWorld.robot + /usr/lib/python3.9/site-packages/xtesting/samples/HelloWorld.robot variable: - 'var01:foo' - 'var02:bar' @@ -66,7 +66,7 @@ tiers: name: behaveframework args: suites: - - /usr/lib/python3.8/site-packages/xtesting/samples/features + - /usr/lib/python3.9/site-packages/xtesting/samples/features tags: - foo - case_name: seventh @@ -97,5 +97,5 @@ tiers: run: name: ansible args: - private_data_dir: /usr/lib/python3.8/site-packages/xtesting/samples + private_data_dir: /usr/lib/python3.9/site-packages/xtesting/samples playbook: helloworld.yml diff --git a/xtesting/core/campaign.py b/xtesting/core/campaign.py index dbde42df..5ca47d6e 100644 --- a/xtesting/core/campaign.py +++ b/xtesting/core/campaign.py @@ -21,11 +21,11 @@ from urllib.parse import urlparse import boto3 from boto3.s3.transfer import TransferConfig import botocore -import pkg_resources import requests from xtesting.core import testcase from xtesting.utils import env +from xtesting.utils import config from xtesting.utils import constants __author__ = "Cedric Ollivier " @@ -120,7 +120,7 @@ class Campaign(): dst_s3_url = os.environ["S3_DST_URL"] multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[ "S3_ENDPOINT_URL"] else 8 * 1024 * 1024 - config = TransferConfig(multipart_threshold=multipart_threshold) + tconfig = TransferConfig(multipart_threshold=multipart_threshold) bucket_name = urlparse(dst_s3_url).netloc s3path = re.search( '^/*(.*)/*$', urlparse(dst_s3_url).path).group(1) @@ -136,7 +136,7 @@ class Campaign(): b3resource.Bucket(bucket_name).download_file( s3_object.key, re.sub(f'^{s3path}/*', '', s3_object.key), - Config=config) + Config=tconfig) Campaign.__logger.info( "Downloading %s", re.sub(f'^{s3path}/*', '', s3_object.key)) @@ -183,7 +183,7 @@ class Campaign(): dst_s3_url = os.environ["S3_DST_URL"] multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[ "S3_ENDPOINT_URL"] else 8 * 1024 * 1024 - config = TransferConfig(multipart_threshold=multipart_threshold) + tconfig = TransferConfig(multipart_threshold=multipart_threshold) bucket_name = urlparse(dst_s3_url).netloc mime_type = mimetypes.guess_type(f'{build_tag}.zip') path = urlparse(dst_s3_url).path.strip("/") @@ -191,7 +191,7 @@ class Campaign(): b3resource.Bucket(bucket_name).upload_file( f'{build_tag}.zip', os.path.join(path, f'{build_tag}.zip'), - Config=config, + Config=tconfig, ExtraArgs={'ContentType': mime_type[ 0] or 'application/octet-stream'}) dst_http_url = os.environ["HTTP_DST_URL"] @@ -217,10 +217,10 @@ def main(): if not os.path.exists(testcase.TestCase.dir_results): os.makedirs(testcase.TestCase.dir_results) if env.get('DEBUG').lower() == 'true': - logging.config.fileConfig(pkg_resources.resource_filename( - 'xtesting', constants.DEBUG_INI_PATH)) + logging.config.fileConfig(config.get_xtesting_config( + 'logging.debug.ini', constants.DEBUG_INI_PATH_DEFAULT)) else: - logging.config.fileConfig(pkg_resources.resource_filename( - 'xtesting', constants.INI_PATH)) + logging.config.fileConfig(config.get_xtesting_config( + 'logging.ini', constants.INI_PATH_DEFAULT)) logging.captureWarnings(True) Campaign.zip_campaign_files() diff --git a/xtesting/utils/config.py b/xtesting/utils/config.py new file mode 100644 index 00000000..6f94fc8f --- /dev/null +++ b/xtesting/utils/config.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# Copyright (c) 2022 Orange and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +# pylint: disable=missing-docstring + +import os + +from xtesting.utils import constants + + +def get_xtesting_config(filename, default): + """Search Xtesting configs (i.e. testcases.yaml)""" + for path in constants.XTESTING_PATHES: + abspath = os.path.abspath(os.path.expanduser(path)) + if os.path.isfile(os.path.join(abspath, filename)): + return os.path.join(abspath, filename) + return default diff --git a/xtesting/utils/constants.py b/xtesting/utils/constants.py index acd0d31d..f5e29e0d 100644 --- a/xtesting/utils/constants.py +++ b/xtesting/utils/constants.py @@ -1,14 +1,33 @@ #!/usr/bin/env python +# Copyright (c) 2019 Orange and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + # pylint: disable=missing-docstring import os +import sys + +import pkg_resources ENV_FILE = '/var/lib/xtesting/conf/env_file' +XTESTING_PATHES = [ + "~/.xtesting", "/etc/xtesting", os.path.join(sys.prefix + "/etc/xtesting")] + +TESTCASE_DESCRIPTION = 'testcases.yaml' +TESTCASE_DESCRIPTION_DEFAULT = pkg_resources.resource_filename( + 'xtesting', f'ci/{TESTCASE_DESCRIPTION}') + RESULTS_DIR = '/var/lib/xtesting/results' LOG_PATH = os.path.join(RESULTS_DIR, 'xtesting.log') DEBUG_LOG_PATH = os.path.join(RESULTS_DIR, 'xtesting.debug.log') -INI_PATH = 'ci/logging.ini' -DEBUG_INI_PATH = 'ci/logging.debug.ini' +INI_PATH_DEFAULT = pkg_resources.resource_filename( + 'xtesting', 'ci/logging.ini') +DEBUG_INI_PATH_DEFAULT = pkg_resources.resource_filename( + 'xtesting', 'ci/logging.debug.ini') -- cgit 1.2.3-korg