diff options
| -rw-r--r-- | dovetail/conf/functest_config.yml | 4 | ||||
| -rw-r--r-- | dovetail/container.py | 14 | ||||
| -rw-r--r-- | dovetail/report.py | 85 | ||||
| -rwxr-xr-x | dovetail/run.py | 4 | ||||
| -rw-r--r-- | dovetail/test_runner.py | 78 | ||||
| -rw-r--r-- | dovetail/testcase.py | 30 | ||||
| -rw-r--r-- | dovetail/testcase/example.tc002.yml | 1 | ||||
| -rw-r--r-- | dovetail/testcase/example.tc003.yml | 14 | ||||
| -rw-r--r-- | dovetail/testcase/ipv6.tc001.yml | 4 | ||||
| -rw-r--r-- | dovetail/tests/unit/test_parser.py | 10 | ||||
| -rw-r--r-- | dovetail/utils/dovetail_utils.py | 69 | ||||
| -rw-r--r-- | ez_setup.py | 427 | ||||
| -rw-r--r-- | requirements.txt | 3 | ||||
| -rw-r--r-- | test-requirements.txt | 4 | ||||
| -rw-r--r-- | tox.ini | 7 | ||||
| -rw-r--r-- | unittests/requirements.txt | 6 | ||||
| -rwxr-xr-x | unittests/unittest.sh | 60 |
17 files changed, 229 insertions, 591 deletions
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml index ceb894b3..6aea2809 100644 --- a/dovetail/conf/functest_config.yml +++ b/dovetail/conf/functest_config.yml @@ -6,12 +6,12 @@ functest: -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal' opts: '-id --privileged=true' pre_condition: - - 'echo test for precondition' + - 'echo test for precondition in functest' cmds: - 'functest env prepare' - 'functest testcase run {{validate_testcase}}' post_condition: - - 'echo test for postcondition' + - 'echo test for postcondition in functest' result: dir: '/home/opnfv/functest/results' store_type: 'file' diff --git a/dovetail/container.py b/dovetail/container.py index 62070442..b3450fa3 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -20,10 +20,10 @@ class Container: logger = None - def __init__(cls): + def __init__(self): pass - def __str__(cls): + def __str__(self): pass @classmethod @@ -67,8 +67,10 @@ class Container: (docker_image)) else: cmd = 'sudo docker pull %s' % (docker_image) - dt_utils.exec_cmd(cmd, cls.logger) - cls.has_pull_latest_image[type] = True + ret, msg = dt_utils.exec_cmd(cmd, cls.logger) + if ret == 0: + cls.logger.debug('docker pull %s success!', docker_image) + cls.has_pull_latest_image[type] = True @classmethod def clean(cls, container_id): @@ -80,6 +82,6 @@ class Container: @classmethod def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): if sub_cmd == "": - return + return (1, 'sub_cmd is empty') cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd) - dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) + return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) diff --git a/dovetail/report.py b/dovetail/report.py index 654e6bf8..8c302b60 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -168,12 +168,13 @@ class Report: type = testcase.validate_type() crawler = CrawlerFactory.create(type) if crawler is None: + cls.logger.error('crawler is None:%s', testcase.name()) return None if validate_testcase in cls.results[type]: return cls.results[type][validate_testcase] - result = crawler.crawl(validate_testcase) + result = crawler.crawl(testcase) if result is not None: cls.results[type][validate_testcase] = result @@ -187,25 +188,13 @@ class Report: return result -class CrawlerFactory: - - @staticmethod - def create(type): - if type == 'functest': - return FunctestCrawler() - - if type == 'yardstick': - return YardstickCrawler() - - return None - - class FunctestCrawler: logger = None def __init__(self): self.type = 'functest' + self.logger.debug('create crawler:%s', self.type) @classmethod def create_log(cls): @@ -258,7 +247,8 @@ class FunctestCrawler: def crawl_from_url(self, testcase=None): url = \ - dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase + dt_cfg.dovetail_config[self.type]['result']['db_url'] % \ + testcase.validate_testcase() self.logger.debug("Query to rest api: %s" % url) try: data = json.load(urllib2.urlopen(url)) @@ -275,6 +265,7 @@ class YardstickCrawler: def __init__(self): self.type = 'yardstick' + self.logger.debug('create crawler:%s', self.type) @classmethod def create_log(cls): @@ -292,7 +283,7 @@ class YardstickCrawler: def crawl_from_file(self, testcase=None): file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], - testcase + '.out') + testcase.validate_testcase() + '.out') if not os.path.exists(file_path): self.logger.info('result file not found: %s' % file_path) return None @@ -312,17 +303,39 @@ class YardstickCrawler: return None -class CheckerFactory: +class ShellCrawler: - @staticmethod - def create(type): - if type == 'functest': - return FunctestChecker() + def __init__(self): + self.type = 'shell' - if type == 'yardstick': - return YardstickChecker() + def crawl(self, testcase=None): + return self.crawl_from_file(testcase) - return None + def crawl_from_file(self, testcase=None): + file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], + testcase.name()) + '.out' + if not os.path.exists(file_path): + return None + try: + with open(file_path, 'r') as json_data: + result = json.load(json_data) + return result + except Exception: + return None + + +class CrawlerFactory: + + CRAWLER_MAP = {'functest': FunctestCrawler, + 'yardstick': YardstickCrawler, + 'shell': ShellCrawler} + + @classmethod + def create(cls, type): + try: + return cls.CRAWLER_MAP[type]() + except KeyError: + return None class ResultChecker: @@ -388,3 +401,27 @@ class YardstickChecker: else: testcase.passed(result['criteria'] == 'PASS') return + + +class ShellChecker: + + @staticmethod + def check(testcase, result): + try: + testcase.passed(result['pass']) + except Exception: + testcase.passed(False) + + +class CheckerFactory: + + CHECKER_MAP = {'functest': FunctestChecker, + 'yardstick': YardstickChecker, + 'shell': ShellChecker} + + @classmethod + def create(cls, type): + try: + return cls.CHECKER_MAP[type]() + except KeyError: + return None diff --git a/dovetail/run.py b/dovetail/run.py index 8a91e4b3..85e65667 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -64,8 +64,8 @@ def run_test(testsuite, testarea, logger): end_time = time.time() duration = end_time - start_time - db_result = Report.get_result(testcase) - Report.check_result(testcase, db_result) + result = Report.get_result(testcase) + Report.check_result(testcase, result) return duration diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py index 8a95b1f7..70dd2345 100644 --- a/dovetail/test_runner.py +++ b/dovetail/test_runner.py @@ -7,8 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +import os +import json import utils.dovetail_utils as dt_utils import utils.dovetail_logger as dt_logger +from utils.dovetail_config import DovetailConfig as dt_cfg from container import Container @@ -19,34 +22,48 @@ class DockerRunner(object): def __init__(self, testcase): self.testcase = testcase + self.logger.debug('create runner: %s', self.type) @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__file__).getLogger() + cls.logger = dt_logger.Logger(__name__ + '.DockerRunner').getLogger() def run(self): Container.pull_image(self.testcase.validate_type()) container_id = Container.create(self.testcase.validate_type()) + if container_id == '': + self.logger.error('failed to create container') + return + self.logger.debug('container id:%s' % container_id) if not self.testcase.prepared(): + failed = False cmds = self.testcase.pre_condition() if cmds: for cmd in cmds: - Container.exec_cmd(container_id, cmd) - self.testcase.prepared(True) + ret, msg = Container.exec_cmd(container_id, cmd) + if ret != 0: + failed = True + break + if not failed: + self.testcase.prepared(True) if not self.testcase.prepare_cmd(): self.logger.error('failed to prepare testcase:%s', self.testcase.name) else: for cmd in self.testcase.cmds: - Container.exec_cmd(container_id, cmd) + ret, msg = Container.exec_cmd(container_id, cmd) + if ret != 0: + self.logger.error('Failed to exec %s, ret:%d, msg:%s', + cmd, ret, msg) + break cmds = self.testcase.post_condition() if cmds: for cmd in cmds: - Container.exec_cmd(container_id, cmd) + ret, msg = Container.exec_cmd(container_id, cmd) self.testcase.cleaned(True) Container.clean(container_id) @@ -55,15 +72,15 @@ class DockerRunner(object): class FunctestRunner(DockerRunner): def __init__(self, testcase): + self.type = 'functest' super(FunctestRunner, self).__init__(testcase) - self.name = 'functest' class YardstickRunner(DockerRunner): def __init__(self, testcase): + self.type = 'yardstick' super(YardstickRunner, self).__init__(testcase) - self.name = 'yardstick' class ShellRunner(object): @@ -72,16 +89,55 @@ class ShellRunner(object): @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__file__).getLogger() + cls.logger = dt_logger.Logger(__name__ + '.ShellRunner').getLogger() def __init__(self, testcase): super(ShellRunner, self).__init__() self.testcase = testcase - self.name = 'shell' + self.type = 'shell' + self.logger.debug('create runner:%s', self.type) def run(self): - for cmd in self.testcase.cmds: - dt_utils.exec_cmd(cmd, self.logger) + passed = True + failed = False + result = {'pass': True, 'results': []} + if not self.testcase.prepared(): + cmds = self.testcase.pre_condition() + for cmd in cmds: + ret, msg = dt_utils.exec_cmd(cmd, self.logger) + result['results'].append((cmd, ret, msg)) + if ret != 0: + failed = True + break + if not failed: + self.testcase.prepared(True) + + if not self.testcase.prepare_cmd(): + self.logger.error('failed to prepare cmd:%s', + self.testcase.name()) + else: + for cmd in self.testcase.cmds: + ret, msg = dt_utils.exec_cmd(cmd, self.logger) + result['results'].append((cmd, ret, msg)) + if ret != 0: + passed = False + + result['pass'] = passed + + cmds = self.testcase.post_condition() + for cmd in cmds: + ret, msg = dt_utils.exec_cmd(cmd, self.logger) + result['results'].append((cmd, ret, msg)) + + result_filename = os.path.join(dt_cfg.dovetail_config['result_dir'], + self.testcase.name()) + '.out' + self.logger.debug('save result:%s', result_filename) + try: + with open(result_filename, 'w') as f: + f.write(json.dumps(result)) + except Exception as e: + self.logger.exception('Failed to write result into file:%s, \ + except:%s', result_filename, e) class TestRunnerFactory(object): diff --git a/dovetail/testcase.py b/dovetail/testcase.py index dd0fd2b5..040c6f98 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -93,17 +93,33 @@ class Testcase(object): return self._result_acquired(self.validate_testcase(), acquired) def pre_condition(self): - return self.pre_condition_cls(self.validate_type()) + try: + pre_condition = self.testcase['validate']['pre_condition'] + if pre_condition == '': + pre_condition = self.pre_condition_cls(self.validate_type()) + return pre_condition + except: + self.logger.debug('testcase:%s pre_condition is empty', + self.name()) + return '' def post_condition(self): - return self.post_condition_cls(self.validate_type()) + try: + post_condition = self.testcase['validate']['post_condition'] + if post_condition == '': + post_condition = self.post_condition_cls(self.validate_type()) + return post_condition + except: + self.logger.debug('testcae:%s post_condition is empty', + self.name()) + return '' def run(self): runner = TestRunnerFactory.create(self) try: runner.run() - except AttributeError: - pass + except AttributeError as e: + self.logger.exception('testcase:%s except:%s', self.name, e) # testcase in upstream testing project # validate_testcase_list = {'functest': {}, 'yardstick': {}, 'shell': {}} @@ -187,7 +203,7 @@ class FunctestTestcase(Testcase): def __init__(self, testcase_yaml): super(FunctestTestcase, self).__init__(testcase_yaml) - self.name = 'functest' + self.type = 'functest' def prepare_cmd(self): ret = super(FunctestTestcase, self).prepare_cmd() @@ -208,7 +224,7 @@ class YardstickTestcase(Testcase): def __init__(self, testcase_yaml): super(YardstickTestcase, self).__init__(testcase_yaml) - self.name = 'yardstick' + self.type = 'yardstick' class ShellTestcase(Testcase): @@ -217,7 +233,7 @@ class ShellTestcase(Testcase): def __init__(self, testcase_yaml): super(ShellTestcase, self).__init__(testcase_yaml) - self.name = 'shell' + self.type = 'shell' class TestcaseFactory(object): diff --git a/dovetail/testcase/example.tc002.yml b/dovetail/testcase/example.tc002.yml index 89d000c9..b9296074 100644 --- a/dovetail/testcase/example.tc002.yml +++ b/dovetail/testcase/example.tc002.yml @@ -8,6 +8,7 @@ dovetail.example.tc002: - "echo pre_condition" cmds: - "echo test2" + - "mkdir xxx" post_condition: - "echo post_condition" report: diff --git a/dovetail/testcase/example.tc003.yml b/dovetail/testcase/example.tc003.yml new file mode 100644 index 00000000..7916e017 --- /dev/null +++ b/dovetail/testcase/example.tc003.yml @@ -0,0 +1,14 @@ +dovetail.example.tc003: + name: dovetail.example.tc003 + objective: doing something useful + validate: + type: shell + testcase: "run shell22" + pre_condition: + cmds: + - "echo test2" + post_condition: + - "" + report: + sub_testcase_list: + diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml index 0bc0baaa..916d0915 100644 --- a/dovetail/testcase/ipv6.tc001.yml +++ b/dovetail/testcase/ipv6.tc001.yml @@ -5,12 +5,12 @@ dovetail.ipv6.tc001: type: functest testcase: tempest_smoke_serial pre_condition: - - 'echo test for precondition' + - 'echo test for precondition in testcase' cmds: - 'functest env prepare' - 'functest testcase run {{validate_testcase}}' post_condition: - - 'echo test for precondition' + - 'echo test for precondition in testcase' report: sub_testcase_list: - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py index 73ed2b72..5fb4dbb9 100644 --- a/dovetail/tests/unit/test_parser.py +++ b/dovetail/tests/unit/test_parser.py @@ -16,7 +16,8 @@ import os import unittest import yaml -import parser as dovetail_parser +from dovetail import parser +from dovetail.utils.dovetail_config import DovetailConfig as dt_cfg class TestParser(unittest.TestCase): @@ -25,9 +26,8 @@ class TestParser(unittest.TestCase): def setUp(self): """Test case setup""" - from utils.dovetail_config import DovetailConfig as dt_cfg dt_cfg.load_config_files() - dovetail_parser.Parser.create_log() + parser.Parser.create_log() logging.disable(logging.CRITICAL) def test_parser_cmd(self): @@ -39,7 +39,7 @@ class TestParser(unittest.TestCase): MockTestcase = type('Testcase', (object,), {}) mock_testcase = MockTestcase() mock_testcase.testcase = mock_testcase_yaml.values()[0] - output = dovetail_parser.Parser.parse_cmd(mock_cmd, mock_testcase) + output = parser.Parser.parse_cmd(mock_cmd, mock_testcase) expected_output = ("python /functest/ci/run_tests.py -t " "tempest_smoke_serial -r") self.assertEqual(expected_output, output) @@ -52,7 +52,7 @@ class TestParser(unittest.TestCase): MockTestcase = type('Testcase', (object,), {}) mock_testcase = MockTestcase() mock_testcase.testcase = mock_testcase_yaml.values() - output = dovetail_parser.Parser.parse_cmd(mock_cmd, mock_testcase) + output = parser.Parser.parse_cmd(mock_cmd, mock_testcase) expected_output = ("python /functest/ci/run_tests.py -t " "None -r") self.assertEqual(expected_output, output) diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index 2d5f0dd1..8c4865d8 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -15,23 +15,34 @@ import subprocess from collections import Mapping, Set, Sequence -def exec_cmd(cmd, logger=None, exit_on_error=True, info=False, - error_msg="", verbose=True): - if not error_msg: - error_msg = ("The command '%s' failed." % cmd) - msg_exec = ("Executing command: '%s'" % cmd) - if verbose: - if logger: - if info: - logger.info(msg_exec) - else: - logger.debug(msg_exec) +def exec_log(verbose, logger, msg, level, flush=False): + if not verbose: + return + + if logger: + if level == 'info': + logger.info(msg) + elif level == 'error': + logger.error(msg) + elif level == 'debug': + logger.debug(msg) else: - print(msg_exec) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + pass + else: + print(msg) + if flush: + sys.stdout.flush() + + +def exec_cmd(cmd, logger=None, exit_on_error=False, info=False, + err_msg="", verbose=True): + msg_err = ("The command '%s' failed." % cmd) if not err_msg else err_msg + msg_exec = ("Executing command: '%s'" % cmd) + level = 'info' if info else 'debug' + exec_log(verbose, logger, msg_exec, level) - # show progress bar + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) seconds = 0 while p.poll() is None: seconds += 1 @@ -39,29 +50,17 @@ def exec_cmd(cmd, logger=None, exit_on_error=True, info=False, show_progress_bar(seconds) time.sleep(1) - output = p.communicate() - for line in output[0].strip().split('\n'): - line = line.replace('\n', '') - if logger: - if info: - logger.info(line) - else: - logger.debug(line) - else: - print (line) - sys.stdout.flush() - - returncode = p.returncode - if returncode != 0: - if verbose: - if logger: - logger.error(error_msg) - else: - print(error_msg) + (stdout, stderr) = p.communicate() + if p.returncode == 0: + for line in stdout.strip().splitlines(): + exec_log(verbose, logger, line, level, True) + else: + exec_log(verbose, logger, stderr, 'error') + exec_log(verbose, logger, msg_err, 'error') if exit_on_error: sys.exit(1) - return returncode, output[0].strip() + return p.returncode, stdout.strip() # walkthrough the object, yield path and value diff --git a/ez_setup.py b/ez_setup.py deleted file mode 100644 index 36da1e62..00000000 --- a/ez_setup.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python - -""" -Setuptools bootstrapping installer. - -Maintained at https://github.com/pypa/setuptools/tree/bootstrap. - -Run this script to install or upgrade setuptools. -""" - -import os -import shutil -import sys -import tempfile -import zipfile -import optparse -import subprocess -import platform -import textwrap -import contextlib -import json -import codecs - -from distutils import log - -try: - from urllib.request import urlopen - from urllib.parse import urljoin -except ImportError: - from urllib2 import urlopen - from urlparse import urljoin - -try: - from site import USER_SITE -except ImportError: - USER_SITE = None - -LATEST = object() -DEFAULT_VERSION = LATEST -DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/" -DEFAULT_SAVE_DIR = os.curdir - - -def _python_cmd(*args): - """ - Execute a command. - - Return True if the command succeeded. - """ - args = (sys.executable,) + args - return subprocess.call(args) == 0 - - -def _install(archive_filename, install_args=()): - """Install Setuptools.""" - with archive_context(archive_filename): - # installing - log.warn('Installing Setuptools') - if not _python_cmd('setup.py', 'install', *install_args): - log.warn('Something went wrong during the installation.') - log.warn('See the error message above.') - # exitcode will be 2 - return 2 - - -def _build_egg(egg, archive_filename, to_dir): - """Build Setuptools egg.""" - with archive_context(archive_filename): - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') - - -class ContextualZipFile(zipfile.ZipFile): - - """Supplement ZipFile class to support context manager for Python 2.6.""" - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __new__(cls, *args, **kwargs): - """Construct a ZipFile or ContextualZipFile as appropriate.""" - if hasattr(zipfile.ZipFile, '__exit__'): - return zipfile.ZipFile(*args, **kwargs) - return super(ContextualZipFile, cls).__new__(cls) - - -@contextlib.contextmanager -def archive_context(filename): - """ - Unzip filename to a temporary directory, set to the cwd. - - The unzipped target is cleaned up after. - """ - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - with ContextualZipFile(filename) as archive: - archive.extractall() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - yield - - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) - - -def _do_download(version, download_base, to_dir, download_delay): - """Download Setuptools.""" - py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys) - tp = 'setuptools-{version}-{py_desig}.egg' - egg = os.path.join(to_dir, tp.format(**locals())) - if not os.path.exists(egg): - archive = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, archive, to_dir) - sys.path.insert(0, egg) - - # Remove previously-imported pkg_resources if present (see - # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). - if 'pkg_resources' in sys.modules: - _unload_pkg_resources() - - import setuptools - setuptools.bootstrap_install_from = egg - - -def use_setuptools( - version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=DEFAULT_SAVE_DIR, download_delay=15): - """ - Ensure that a setuptools version is installed. - - Return None. Raise SystemExit if the requested version - or later cannot be installed. - """ - version = _resolve_version(version) - to_dir = os.path.abspath(to_dir) - - # prior to importing, capture the module state for - # representative modules. - rep_modules = 'pkg_resources', 'setuptools' - imported = set(sys.modules).intersection(rep_modules) - - try: - import pkg_resources - pkg_resources.require("setuptools>=" + version) - # a suitable version is already installed - return - except ImportError: - # pkg_resources not available; setuptools is not installed; download - pass - except pkg_resources.DistributionNotFound: - # no version of setuptools was found; allow download - pass - except pkg_resources.VersionConflict as VC_err: - if imported: - _conflict_bail(VC_err, version) - - # otherwise, unload pkg_resources to allow the downloaded version to - # take precedence. - del pkg_resources - _unload_pkg_resources() - - return _do_download(version, download_base, to_dir, download_delay) - - -def _conflict_bail(VC_err, version): - """ - Setuptools was imported prior to invocation, so it is - unsafe to unload it. Bail out. - """ - conflict_tmpl = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. - - (Currently using {VC_err.args[0]!r}) - """) - msg = conflict_tmpl.format(**locals()) - sys.stderr.write(msg) - sys.exit(2) - - -def _unload_pkg_resources(): - sys.meta_path = [ - importer - for importer in sys.meta_path - if importer.__class__.__module__ != 'pkg_resources.extern' - ] - del_modules = [ - name for name in sys.modules - if name.startswith('pkg_resources') - ] - for mod_name in del_modules: - del sys.modules[mod_name] - - -def _clean_check(cmd, target): - """ - Run the command to download target. - - If the command fails, clean up before re-raising the error. - """ - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if os.access(target, os.F_OK): - os.unlink(target) - raise - - -def download_file_powershell(url, target): - """ - Download the file at url to target using Powershell. - - Powershell will validate trust. - Raise an exception if the command cannot complete. - """ - target = os.path.abspath(target) - ps_cmd = ( - "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " - "[System.Net.CredentialCache]::DefaultCredentials; " - '(new-object System.Net.WebClient).DownloadFile("%(url)s", \ - "%(target)s")' - % locals() - ) - cmd = [ - 'powershell', - '-Command', - ps_cmd, - ] - _clean_check(cmd, target) - - -def has_powershell(): - """Determine if Powershell is available.""" - if platform.system() != 'Windows': - return False - cmd = ['powershell', '-Command', 'echo test'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True -download_file_powershell.viable = has_powershell - - -def download_file_curl(url, target): - cmd = ['curl', url, '--location', '--silent', '--output', target] - _clean_check(cmd, target) - - -def has_curl(): - cmd = ['curl', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True -download_file_curl.viable = has_curl - - -def download_file_wget(url, target): - cmd = ['wget', url, '--quiet', '--output-document', target] - _clean_check(cmd, target) - - -def has_wget(): - cmd = ['wget', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True -download_file_wget.viable = has_wget - - -def download_file_insecure(url, target): - """Use Python to download the file, without connection authentication.""" - src = urlopen(url) - try: - # Read all the data in one block. - data = src.read() - finally: - src.close() - - # Write all the data in one block to avoid creating a partial file. - with open(target, "wb") as dst: - dst.write(data) -download_file_insecure.viable = lambda: True - - -def get_best_downloader(): - downloaders = ( - download_file_powershell, - download_file_curl, - download_file_wget, - download_file_insecure, - ) - viable_downloaders = (dl for dl in downloaders if dl.viable()) - return next(viable_downloaders, None) - - -def download_setuptools( - version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=DEFAULT_SAVE_DIR, delay=15, - downloader_factory=get_best_downloader): - """ - Download setuptools from a specified location and return its filename. - - `version` should be a valid setuptools version number that is available - as an sdist for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download - attempt. - - ``downloader_factory`` should be a function taking no arguments and - returning a function for downloading a URL to a target. - """ - version = _resolve_version(version) - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - zip_name = "setuptools-%s.zip" % version - url = download_base + zip_name - saveto = os.path.join(to_dir, zip_name) - if not os.path.exists(saveto): # Avoid repeated downloads - log.warn("Downloading %s", url) - downloader = downloader_factory() - downloader(url, saveto) - return os.path.realpath(saveto) - - -def _resolve_version(version): - """ - Resolve LATEST version - """ - if version is not LATEST: - return version - - meta_url = urljoin(DEFAULT_URL, '/pypi/setuptools/json') - resp = urlopen(meta_url) - with contextlib.closing(resp): - try: - charset = resp.info().get_content_charset() - except Exception: - # Python 2 compat; assume UTF-8 - charset = 'UTF-8' - reader = codecs.getreader(charset) - doc = json.load(reader(resp)) - - return str(doc['info']['version']) - - -def _build_install_args(options): - """ - Build the arguments to 'python setup.py install' on the setuptools package. - - Returns list of command line arguments. - """ - return ['--user'] if options.user_install else [] - - -def _parse_args(): - """Parse the command line for options.""" - parser = optparse.OptionParser() - parser.add_option( - '--user', dest='user_install', action='store_true', default=False, - help='install in user site package') - parser.add_option( - '--download-base', dest='download_base', metavar="URL", - default=DEFAULT_URL, - help='alternative URL from where to download the setuptools package') - parser.add_option( - '--insecure', dest='downloader_factory', action='store_const', - const=lambda: download_file_insecure, default=get_best_downloader, - help='Use internal, non-validating downloader' - ) - parser.add_option( - '--version', help="Specify which version to download", - default=DEFAULT_VERSION, - ) - parser.add_option( - '--to-dir', - help="Directory to save (and re-use) package", - default=DEFAULT_SAVE_DIR, - ) - options, args = parser.parse_args() - # positional arguments are ignored - return options - - -def _download_args(options): - """Return args for download_setuptools function from cmdline args.""" - return dict( - version=options.version, - download_base=options.download_base, - downloader_factory=options.downloader_factory, - to_dir=options.to_dir, - ) - - -def main(): - """Install or upgrade setuptools and EasyInstall.""" - options = _parse_args() - archive = download_setuptools(**_download_args(options)) - return _install(archive, _build_install_args(options)) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index a176ad61..60f43d71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,3 @@ -pbr>=1.6 -coverage>=3.6 -flake8<3.0 Jinja2>=2.6 PyYAML>=3.10 Click diff --git a/test-requirements.txt b/test-requirements.txt index 8ea9c2b6..2796798a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,7 @@ +coverage>=3.6 +pep8==1.5.7 +pyflakes==0.8.1 +flake8<3.0 pytest pykwalify mock @@ -1,7 +1,7 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py27,pep8 +envlist = py27 [testenv] basepython=python2 @@ -20,6 +20,11 @@ setenv = deps = flake8 commands = flake8 {toxinidir} +[testenv:py27-cover] +commands = python setup.py testr --coverage \ + --omit='{toxinidir}/dovetail/tests*' \ + --testr-args='{posargs}' + [flake8] show-source = True ignore = E123,E125,H803 diff --git a/unittests/requirements.txt b/unittests/requirements.txt deleted file mode 100644 index 694a62b4..00000000 --- a/unittests/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -Click -coverage==4.1b2 -flake8==2.5.4 -Jinja2==2.8 -PyYAML==3.11 - diff --git a/unittests/unittest.sh b/unittests/unittest.sh deleted file mode 100755 index 215d0ebd..00000000 --- a/unittests/unittest.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -############################################################################## -# Copyright (c) 2016 OPNFV and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# Run flake8 -# Future, more can be added when necessary, such as coverage/functional/unit check -# of the dovetail tool - -getopts ":f" FILE_OPTION - -run_flake8() { - echo "Running flake8 ... " - logfile=unittest_results.log - if [ $FILE_OPTION == "f" ]; then - flake8 dovetail > $logfile - else - flake8 dovetail - fi - - if [ $? -ne 0 ]; then - echo "FAILED" - if [ $FILE_OPTION == "f" ]; then - echo "Results in $logfile" - fi - exit 1 - else - echo "OK" - fi -} - -run_tests() { - echo "Running unittest ..." - cd dovetail/ - if [ $FILE_OPTION == "f" ]; then - python -m unittest discover -v -s tests/unit > $logfile 2>&1 - else - python -m unittest discover -v -s tests/unit - fi - - if [ $? -ne 0 ]; then - if [ $FILE_OPTION == "f" ]; then - echo "FAILED, results in $logfile" - fi - exit 1 - else - if [ $FILE_OPTION == "f" ]; then - echo "OK, results in $logfile" - fi - fi -} - -run_flake8 -run_tests |