diff options
| -rw-r--r-- | docker/components/testcases.yaml | 27 | ||||
| -rw-r--r-- | docker/smoke/Dockerfile | 5 | ||||
| -rw-r--r-- | docker/smoke/testcases.yaml | 63 | ||||
| -rw-r--r-- | docker/tempest/Dockerfile | 13 | ||||
| -rw-r--r-- | functest/ci/testcases.yaml | 90 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 24 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/shaker/shaker.py | 5 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/tempest.py | 27 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/vmtp/vmtp.py | 2 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/rally/test_rally.py | 15 | ||||
| -rw-r--r-- | upper-constraints.txt | 1 |
11 files changed, 210 insertions, 62 deletions
diff --git a/docker/components/testcases.yaml b/docker/components/testcases.yaml index babe3e687..6292d339c 100644 --- a/docker/components/testcases.yaml +++ b/docker/components/testcases.yaml @@ -9,17 +9,38 @@ tiers: - case_name: tempest_full project_name: functest - criteria: 80 + criteria: 100 + blocking: false + description: >- + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' + + - + case_name: tempest_scenario + project_name: functest + criteria: 100 blocking: false description: >- The list of test cases is generated by Tempest automatically and depends on the parameters of - the OpenStack deplopyment. + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestCommon' args: - mode: '^(tempest|neutron_tempest_plugin)\.' + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' + option: + - '--concurrency=1' - case_name: rally_full diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile index ff905e31b..1ed39bdc0 100644 --- a/docker/smoke/Dockerfile +++ b/docker/smoke/Dockerfile @@ -4,7 +4,7 @@ ARG BRANCH=master ARG OPENSTACK_TAG=stable/queens ARG REFSTACK_TARGET=2017.09 ARG PATROLE_TAG=0.3.0 -ARG VMTP_TAG=refs/changes/59/588459/1 +ARG VMTP_TAG=99b261ccccc2f8a08ee2d8fca9f54ef9d69899d7 RUN apk --no-cache add --update libxml2 libxslt && \ apk --no-cache add --virtual .build-deps --update \ @@ -26,7 +26,8 @@ RUN apk --no-cache add --update libxml2 libxslt && \ git clone --depth 1 https://github.com/openstack/barbican-tempest-plugin.git /src/barbican-tempest-plugin && \ update-requirements -s --source /src/openstack-requirements /src/barbican-tempest-plugin/ && \ pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ - /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin /src/vmtp && \ + /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \ + networking-bgpvpn networking-sfc /src/vmtp && \ virtualenv --system-site-packages /src/tempest/.venv && \ rm -r upper-constraints.txt upper-constraints.opnfv.txt \ /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin /src/vmtp && \ diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml index 20112a68e..65190f414 100644 --- a/docker/smoke/testcases.yaml +++ b/docker/smoke/testcases.yaml @@ -16,12 +16,32 @@ tiers: Tempest suite. The list of test cases is generated by Tempest automatically and depends on the parameters of the OpenStack deplopyment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestCommon' args: - mode: - '^(tempest|neutron_tempest_plugin)\.(api|scenario).*\[.*\bsmoke\b.*\]$' + mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' + + - + case_name: neutron-tempest-plugin-api + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the Tempest suite proposed by the + Neutron project. The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '^neutron_tempest_plugin\.api' + option: + - '--concurrency=4' - case_name: rally_sanity @@ -135,6 +155,45 @@ tiers: - trunk-details - + case_name: networking-bgpvpn + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs OpenStack networking-bgpvpn Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '^networking_bgpvpn_tempest\.' + neutron_extensions: + - bgpvpn + option: + - '--concurrency=4' + + - + case_name: networking-sfc + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs OpenStack networking-sfc Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: + '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$' + neutron_extensions: + - sfc + option: + - '--concurrency=0' + + - case_name: barbican project_name: functest criteria: 100 diff --git a/docker/tempest/Dockerfile b/docker/tempest/Dockerfile index 1d46955fe..a09a562fd 100644 --- a/docker/tempest/Dockerfile +++ b/docker/tempest/Dockerfile @@ -8,9 +8,16 @@ ARG OS_FAULTS_TAG=0.1.17 RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/github.com\\/openstack\\/tempest@\\1#egg=tempest/ \ - > upper-constraints.txt && \ + case $OPENSTACK_TAG in \ + master) \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ + sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/github.com\\/openstack\\/tempest#egg=tempest/ \ + > upper-constraints.txt ;; \ + *) \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ + sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/github.com\\/openstack\\/tempest@\\1#egg=tempest/ \ + > upper-constraints.txt ;; \ + esac && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 511935e6b..bff4da191 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -207,12 +207,32 @@ tiers: Tempest suite. The list of test cases is generated by Tempest automatically and depends on the parameters of the OpenStack deplopyment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestCommon' args: - mode: - '^(tempest|neutron_tempest_plugin)\.(api|scenario).*\[.*\bsmoke\b.*\]$' + mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' + + - + case_name: neutron-tempest-plugin-api + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the Tempest suite proposed by the + Neutron project. The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '^neutron_tempest_plugin\.api' + option: + - '--concurrency=4' - case_name: rally_sanity @@ -326,6 +346,45 @@ tiers: - trunk-details - + case_name: networking-bgpvpn + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs OpenStack networking-bgpvpn Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '^networking_bgpvpn_tempest\.' + neutron_extensions: + - bgpvpn + option: + - '--concurrency=4' + + - + case_name: networking-sfc + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs OpenStack networking-sfc Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: + '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$' + neutron_extensions: + - sfc + option: + - '--concurrency=0' + + - case_name: barbican project_name: functest criteria: 100 @@ -434,17 +493,38 @@ tiers: - case_name: tempest_full project_name: functest - criteria: 80 + criteria: 100 blocking: false description: >- The list of test cases is generated by Tempest automatically and depends on the parameters of - the OpenStack deplopyment. + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' + + - + case_name: tempest_scenario + project_name: functest + criteria: 100 + blocking: false + description: >- + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestCommon' args: - mode: '^(tempest|neutron_tempest_plugin)\.' + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' + option: + - '--concurrency=1' - case_name: rally_full diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index efee15aee..4a6362903 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -169,14 +169,6 @@ class RallyBase(singlevm.VmReady1): return False @staticmethod - def get_cmd_output(proc): - """Get command stdout.""" - result = "" - for line in proc.stdout: - result += line - return result - - @staticmethod def excl_scenario(): """Exclude scenario.""" black_tests = [] @@ -345,23 +337,15 @@ class RallyBase(singlevm.VmReady1): task_file, "--task-args", str(self._build_task_args(test_name))]) LOGGER.debug('running command: %s', cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = self.get_cmd_output(proc) - task_id = self.get_task_id(output) + output = proc.communicate()[0] + task_id = self.get_task_id(output) LOGGER.debug('task_id : %s', task_id) - if task_id is None: - LOGGER.error('Failed to retrieve task_id, validating task...') - cmd = (["rally", "task", "validate", "--task", task_file, - "--task-args", str(self._build_task_args(test_name))]) - LOGGER.debug('running command: %s', cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = self.get_cmd_output(proc) - LOGGER.error("Task validation result:" + "\n" + output) + LOGGER.error("Failed to retrieve task_id") + LOGGER.error("Result:\n%s", output) raise Exception("Failed to retrieve task id") self._save_results(test_name, task_id) diff --git a/functest/opnfv_tests/openstack/shaker/shaker.py b/functest/opnfv_tests/openstack/shaker/shaker.py index bd0f51c50..c005df055 100644 --- a/functest/opnfv_tests/openstack/shaker/shaker.py +++ b/functest/opnfv_tests/openstack/shaker/shaker.py @@ -38,6 +38,7 @@ class Shaker(singlevm.SingleVm2): port = 9000 ssh_connect_loops = 12 create_server_timeout = 300 + shaker_timeout = '3600' def __init__(self, **kwargs): super(Shaker, self).__init__(**kwargs) @@ -90,7 +91,7 @@ class Shaker(singlevm.SingleVm2): 'export OS_PASSWORD={} && ' '{}' 'env && ' - 'shaker --image-name {} --flavor-name {} ' + 'timeout {} shaker --image-name {} --flavor-name {} ' '--server-endpoint {}:9000 --scenario ' 'openstack/full_l2,' 'openstack/full_l3_east_west,' @@ -101,7 +102,7 @@ class Shaker(singlevm.SingleVm2): self.project.project.id, self.project.password, 'export OS_CACERT=~/os_cacert && ' if os.environ.get( 'OS_CACERT') else '', - self.image.name, self.flavor.name, + self.shaker_timeout, self.image.name, self.flavor.name, self.fip.floating_ip_address)) self.__logger.info("output:\n%s", stdout.read()) self.__logger.info("error:\n%s", stderr.read()) diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 05f13c051..452c82d26 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -112,10 +112,10 @@ class TempestCommon(singlevm.VmReady1): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in proc.stdout: + LOGGER.info(line.rstrip()) new_line = line.replace(' ', '').split('|') if 'Tests' in new_line: break - LOGGER.info(line) if 'Testscount' in new_line: result['num_tests'] = int(new_line[2]) elif 'Success' in new_line: @@ -204,32 +204,27 @@ class TempestCommon(singlevm.VmReady1): f_stdout = open( os.path.join(self.res_dir, "tempest.log"), 'w+') - f_stderr = open( - os.path.join(self.res_dir, - "tempest-error.log"), 'w+') proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, - stderr=f_stderr, + stderr=subprocess.STDOUT, bufsize=1) with proc.stdout: for line in iter(proc.stdout.readline, b''): if re.search(r"\} tempest\.", line): - LOGGER.info(line.replace('\n', '')) + LOGGER.info(line.rstrip()) elif re.search(r'(?=\(UUID=(.*)\))', line): self.verification_id = re.search( r'(?=\(UUID=(.*)\))', line).group(1) - LOGGER.info('Verification UUID: %s', self.verification_id) f_stdout.write(line) proc.wait() - f_stdout.close() - f_stderr.close() if self.verification_id is None: raise Exception('Verification UUID not found') + LOGGER.info('Verification UUID: %s', self.verification_id) def parse_verifier_result(self): """Parse and save test results.""" @@ -247,7 +242,7 @@ class TempestCommon(singlevm.VmReady1): return with open(os.path.join(self.res_dir, - "tempest-error.log"), 'r') as logfile: + "tempest.log"), 'r') as logfile: output = logfile.read() success_testcases = [] @@ -309,6 +304,17 @@ class TempestCommon(singlevm.VmReady1): with open(rally_conf, 'wb') as config_file: rconfig.write(config_file) + def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'): + """Print rally logs in res dir""" + if not os.path.exists(self.res_dir): + os.makedirs(self.res_dir) + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + rconfig.set('DEFAULT', 'log-file', 'rally.log') + rconfig.set('DEFAULT', 'log_dir', self.res_dir) + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + def configure(self, **kwargs): # pylint: disable=unused-argument """ Create all openstack resources for tempest-based testcases and write @@ -339,6 +345,7 @@ class TempestCommon(singlevm.VmReady1): **kwargs) == testcase.TestCase.EX_OK self.update_rally_regex() self.update_default_role() + self.update_rally_logs() self.configure(**kwargs) self.generate_test_list(**kwargs) self.apply_tempest_blacklist() diff --git a/functest/opnfv_tests/openstack/vmtp/vmtp.py b/functest/opnfv_tests/openstack/vmtp/vmtp.py index aada034da..9e6fd8e15 100644 --- a/functest/opnfv_tests/openstack/vmtp/vmtp.py +++ b/functest/opnfv_tests/openstack/vmtp/vmtp.py @@ -58,7 +58,7 @@ class Vmtp(singlevm.VmReady2): (_, self.privkey_filename) = tempfile.mkstemp() (_, self.pubkey_filename) = tempfile.mkstemp() - def create_network_ressources(self): + def create_network_resources(self): """Create router It creates a router which gateway is the external network detected. diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index b49fe627a..1a43e561d 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -104,12 +104,6 @@ class OSRallyTesting(unittest.TestCase): self.assertEqual(self.rally_base.task_succeed(json_raw), True) - def test_get_cmd_output(self): - proc = mock.Mock() - proc.stdout.__iter__ = mock.Mock(return_value=iter(['line1', 'line2'])) - self.assertEqual(self.rally_base.get_cmd_output(proc), - 'line1line2') - @mock.patch('six.moves.builtins.open', mock.mock_open()) @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load', return_value={'scenario': [ @@ -222,8 +216,6 @@ class OSRallyTesting(unittest.TestCase): '_build_task_args', return_value={}) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'get_task_id', return_value=None) - @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - 'get_cmd_output', return_value='') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists', return_value=True) @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen') @@ -232,7 +224,7 @@ class OSRallyTesting(unittest.TestCase): # pylint: disable=unused-argument with self.assertRaises(Exception): self.rally_base._run_task('test_name') - text = 'Failed to retrieve task_id, validating task...' + text = 'Failed to retrieve task_id' mock_logger_error.assert_any_call(text) @mock.patch('six.moves.builtins.open', mock.mock_open()) @@ -245,8 +237,6 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'get_task_id', return_value='1') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - 'get_cmd_output', return_value='') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'task_succeed', return_value=True) @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists', return_value=True) @@ -264,12 +254,9 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('six.moves.builtins.open', mock.mock_open()) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'task_succeed', return_value=True) - @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - 'get_cmd_output', return_value='') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists', return_value=True) @mock.patch('subprocess.check_output') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info') @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug') diff --git a/upper-constraints.txt b/upper-constraints.txt index bbd08b644..0122580fe 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -19,4 +19,5 @@ robotframework-sshlibrary===2.1.3;python_version=='2.7' ansible===2.3.2.0 xtesting===0.55.0 networking-bgpvpn===8.0.0 +networking-sfc===6.0.0 sphinx-opnfv-theme===0.1.1 |