summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--ci/run.yaml2
-rw-r--r--docker/smoke/Dockerfile6
-rw-r--r--docker/smoke/testcases.yaml11
-rw-r--r--docs/release/release-notes/functest-release.rst1
-rw-r--r--functest/ci/config_functest.yaml1
-rw-r--r--functest/ci/testcases.yaml11
-rw-r--r--functest/opnfv_tests/openstack/rally/blacklist.txt5
-rw-r--r--functest/opnfv_tests/openstack/rally/rally.py134
-rw-r--r--functest/opnfv_tests/openstack/rally/rally_jobs.yaml3
-rw-r--r--functest/tests/unit/openstack/rally/test_rally.py10
-rw-r--r--requirements.txt1
-rw-r--r--setup.cfg1
-rw-r--r--tox.ini1
13 files changed, 170 insertions, 17 deletions
diff --git a/ci/run.yaml b/ci/run.yaml
index 9cc374c0c..385d38fb1 100644
--- a/ci/run.yaml
+++ b/ci/run.yaml
@@ -66,6 +66,7 @@
- tempest_smoke
- neutron-tempest-plugin-api
- rally_sanity
+ - rally_jobs
- refstack_defcore
- patrole
- snaps_smoke
@@ -86,6 +87,7 @@
- name: '{repo}-functest-smoke-tempest_smoke-{tag}-run'
- name: '{repo}-functest-smoke-neutron-tempest-plugin-api-{tag}-run'
- name: '{repo}-functest-smoke-rally_sanity-{tag}-run'
+ - name: '{repo}-functest-smoke-rally_jobs-{tag}-run'
- name: '{repo}-functest-smoke-refstack_defcore-{tag}-run'
- name: '{repo}-functest-smoke-patrole-{tag}-run'
- name: '{repo}-functest-smoke-snaps_smoke-{tag}-run'
diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile
index d60816834..2f729c646 100644
--- a/docker/smoke/Dockerfile
+++ b/docker/smoke/Dockerfile
@@ -6,6 +6,7 @@ ARG REFSTACK_TARGET=2018.02
ARG PATROLE_TAG=0.3.0
ARG NEUTRON_TAG=0.2.0
ARG BARBICAN_TAG=0.1.0
+ARG NEUTRON_RALLY_TAG=master
RUN apk --no-cache add --virtual .build-deps --update \
python-dev build-base linux-headers libffi-dev \
@@ -41,6 +42,11 @@ RUN apk --no-cache add --virtual .build-deps --update \
mkdir -p /etc/neutron /etc/glance && \
wget -q -O /etc/neutron/policy.json https://git.openstack.org/cgit/openstack/neutron/plain/etc/policy.json?h=$OPENSTACK_TAG && \
wget -q -O /etc/glance/policy.json https://git.openstack.org/cgit/openstack/glance/plain/etc/policy.json?h=$OPENSTACK_TAG && \
+ git clone --depth 1 -b $NEUTRON_RALLY_TAG https://git.openstack.org/openstack/neutron.git /src/neutron && \
+ (cd /src/neutron && git fetch --tags origin $NEUTRON_RALLY_TAG && git checkout FETCH_HEAD) && \
+ mkdir -p /home/opnfv/functest/data/rally/neutron && \
+ cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
+ rm -r /src/neutron && \
apk del .build-deps
COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index a510c6474..381791ac6 100644
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -53,6 +53,17 @@ tiers:
name: rally_sanity
-
+ case_name: rally_jobs
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs a group of Rally jobs used in
+ OpenStack gating
+ run:
+ name: rally_jobs
+
+ -
case_name: refstack_defcore
project_name: functest
criteria: 100
diff --git a/docs/release/release-notes/functest-release.rst b/docs/release/release-notes/functest-release.rst
index ad04be68b..57a2fcf47 100644
--- a/docs/release/release-notes/functest-release.rst
+++ b/docs/release/release-notes/functest-release.rst
@@ -131,6 +131,7 @@ New test cases
* singlevm2
* cinder_test
* neutron-tempest-plugin-api
+ * rally_jobs
* networking-bgpvpn
* networking-sfc
* barbican
diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml
index 557a67b94..dbea320c9 100644
--- a/functest/ci/config_functest.yaml
+++ b/functest/ci/config_functest.yaml
@@ -11,6 +11,7 @@ general:
functest_conf: /home/opnfv/functest/conf
functest_data: /home/opnfv/functest/data
ims_data: /home/opnfv/functest/data/ims/
+ rally_data: /home/opnfv/functest/data/rally
refstack_data: /home/opnfv/functest/data/refstack
router_data: /home/opnfv/functest/data/router/opnfv-vnf-data
functest_images: /home/opnfv/functest/images
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 2413287ac..8f113ba93 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -225,6 +225,17 @@ tiers:
name: rally_sanity
-
+ case_name: rally_jobs
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs a group of Rally jobs used in
+ OpenStack gating
+ run:
+ name: rally_jobs
+
+ -
case_name: refstack_defcore
project_name: functest
criteria: 100
diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt
index 0900f21c1..1ade43eaf 100644
--- a/functest/opnfv_tests/openstack/rally/blacklist.txt
+++ b/functest/opnfv_tests/openstack/rally/blacklist.txt
@@ -17,3 +17,8 @@ functionality:
- NovaServers.boot_server_attach_created_volume_and_live_migrate
- NovaServers.boot_server_from_volume_and_live_migrate
- NovaServers.boot_and_migrate_server
+ -
+ functions:
+ - no_net_trunk_service
+ tests:
+ - '^NeutronTrunk'
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index 00e759150..0f74896d7 100644
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -16,11 +16,13 @@ import json
import logging
import os
import re
+import shutil
import subprocess
import time
import pkg_resources
import prettytable
+from ruamel.yaml import YAML
from six.moves import configparser
from xtesting.core import testcase
from xtesting.energy import energy
@@ -53,7 +55,6 @@ class RallyBase(singlevm.VmReady2):
TENANTS_AMOUNT = 3
ITERATIONS_AMOUNT = 10
CONCURRENCY = 4
- RESULTS_DIR = os.path.join(getattr(config.CONF, 'dir_results'), 'rally')
BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
TEMP_DIR = os.path.join(RALLY_DIR, "var")
@@ -75,6 +76,9 @@ class RallyBase(singlevm.VmReady2):
role_name, user=self.project.user.id,
project=self.project.project.id,
domain=self.project.domain.id)
+ self.results_dir = os.path.join(
+ getattr(config.CONF, 'dir_results'), self.case_name)
+ self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
self.creators = []
self.summary = []
self.scenario_dir = ''
@@ -86,8 +90,8 @@ class RallyBase(singlevm.VmReady2):
self.compute_cnt = 0
self.flavor_alt = None
self.tests = []
- self.task_file = ''
self.run_cmd = ''
+ self.network_extensions = []
def _build_task_args(self, test_file_name):
"""Build arguments for the Rally task."""
@@ -138,7 +142,7 @@ class RallyBase(singlevm.VmReady2):
if not os.path.exists(self.TEMP_DIR):
os.makedirs(self.TEMP_DIR)
- self._apply_blacklist(scenario_file_name, test_file_name)
+ self.apply_blacklist(scenario_file_name, test_file_name)
return test_file_name
@staticmethod
@@ -204,7 +208,12 @@ class RallyBase(singlevm.VmReady2):
"""Determine if migration is supported."""
if self.compute_cnt > 1:
return True
+ return False
+ def _network_trunk_supported(self):
+ """Determine if network trunk service is available"""
+ if 'trunk' in self.network_extensions:
+ return True
return False
@staticmethod
@@ -261,6 +270,8 @@ class RallyBase(singlevm.VmReady2):
if not self._migration_supported():
func_list.append("no_migration")
+ if not self._network_trunk_supported():
+ func_list.append("no_net_trunk_service")
if 'functionality' in black_list_yaml.keys():
for item in black_list_yaml['functionality']:
@@ -274,7 +285,7 @@ class RallyBase(singlevm.VmReady2):
return black_tests
- def _apply_blacklist(self, case_file_name, result_file_name):
+ def apply_blacklist(self, case_file_name, result_file_name):
"""Apply blacklist."""
LOGGER.debug("Applying blacklist...")
cases_file = open(case_file_name, 'r')
@@ -317,10 +328,10 @@ class RallyBase(singlevm.VmReady2):
def _save_results(self, test_name, task_id):
""" Generate and save task execution results"""
# check for result directory and create it otherwise
- if not os.path.exists(self.RESULTS_DIR):
+ if not os.path.exists(self.results_dir):
LOGGER.debug('%s does not exist, we create it.',
- self.RESULTS_DIR)
- os.makedirs(self.RESULTS_DIR)
+ self.results_dir)
+ os.makedirs(self.results_dir)
# put detailed result to log
cmd = (["rally", "task", "detailed", "--uuid", task_id])
@@ -329,8 +340,8 @@ class RallyBase(singlevm.VmReady2):
LOGGER.info("%s\n%s", " ".join(cmd), output)
# save report as JSON
- report_json_name = 'opnfv-{}.json'.format(test_name)
- report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
+ report_json_name = '{}.json'.format(test_name)
+ report_json_dir = os.path.join(self.results_dir, report_json_name)
cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
"--out", report_json_dir])
LOGGER.debug('running command: %s', cmd)
@@ -338,8 +349,8 @@ class RallyBase(singlevm.VmReady2):
LOGGER.info("%s\n%s", " ".join(cmd), output)
# save report as HTML
- report_html_name = 'opnfv-{}.html'.format(test_name)
- report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
+ report_html_name = '{}.html'.format(test_name)
+ report_html_dir = os.path.join(self.results_dir, report_html_name)
cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
"--out", report_html_dir])
LOGGER.debug('running command: %s', cmd)
@@ -409,7 +420,6 @@ class RallyBase(singlevm.VmReady2):
else:
raise Exception("Test name '%s' is invalid" % self.test_name)
- self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
if not os.path.exists(self.task_file):
LOGGER.error("Task file '%s' does not exist.", self.task_file)
raise Exception("Task file '{}' does not exist.".
@@ -417,6 +427,7 @@ class RallyBase(singlevm.VmReady2):
self.update_keystone_default_role()
self.compute_cnt = len(self.cloud.list_hypervisors())
+ self.network_extensions = self.cloud.get_network_extensions()
self.flavor_alt = self.create_flavor_alt()
LOGGER.debug("flavor: %s", self.flavor_alt)
@@ -464,7 +475,7 @@ class RallyBase(singlevm.VmReady2):
except ZeroDivisionError:
success_avg = 0
success_str = str("{:0.2f}".format(success_avg)) + '%'
- duration_str = time.strftime("%M:%S",
+ duration_str = time.strftime("%H:%M:%S",
time.gmtime(item['overall_duration']))
res_table.add_row([item['test_name'], duration_str,
item['nb_tests'], success_str])
@@ -559,3 +570,100 @@ class RallyFull(RallyBase):
self.test_name = 'all'
self.smoke = False
self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
+
+
+class RallyJobs(RallyBase):
+ """Rally OpenStack CI testcase implementation."""
+
+ TESTS = ["neutron"]
+
+ def __init__(self, **kwargs):
+ """Initialize RallyJobs object."""
+ if "case_name" not in kwargs:
+ kwargs["case_name"] = "rally_jobs"
+ super(RallyJobs, self).__init__(**kwargs)
+ self.test_name = 'all'
+ self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
+ self.task_yaml = None
+
+ def prepare_run(self):
+ """Create resources needed by test scenarios."""
+ super(RallyJobs, self).prepare_run()
+ with open(os.path.join(self.RALLY_DIR,
+ 'rally_jobs.yaml'), 'r') as task_file:
+ self.task_yaml = yaml.safe_load(task_file)
+
+ if not all(task in self.task_yaml for task in self.tests):
+ raise Exception("Test '%s' not in '%s'" %
+ (self.test_name, self.tests))
+
+ def apply_blacklist(self, case_file_name, result_file_name):
+ # pylint: disable=too-many-branches
+ """Apply blacklist."""
+ LOGGER.debug("Applying blacklist...")
+ black_tests = list(set(self.excl_func() +
+ self.excl_scenario()))
+ if black_tests:
+ LOGGER.debug("Blacklisted tests: %s", str(black_tests))
+
+ template = YAML(typ='jinja2')
+ with open(case_file_name, 'r') as fname:
+ cases = template.load(fname)
+ if cases.get("version", 1) == 1:
+ # scenarios in dictionary
+ for name in cases.keys():
+ if self.in_iterable_re(name, black_tests):
+ cases.pop(name)
+ else:
+ # workloads in subtasks
+ for sind, subtask in enumerate(cases.get('subtasks', [])):
+ idx = []
+ for wind, workload in enumerate(subtask.get('workloads', [])):
+ scenario = workload.get('scenario', {})
+ for name in scenario.keys():
+ if self.in_iterable_re(name, black_tests):
+ idx.append(wind)
+ break
+ for wind in reversed(idx):
+ cases['subtasks'][sind]['workloads'].pop(wind)
+ # scenarios in subtasks
+ idx = []
+ for sind, subtask in enumerate(cases.get('subtasks', [])):
+ scenario = subtask.get('scenario', {})
+ for name in scenario.keys():
+ if self.in_iterable_re(name, black_tests):
+ idx.append(sind)
+ break
+ for sind in reversed(idx):
+ cases['subtasks'].pop(sind)
+
+ with open(result_file_name, 'w') as fname:
+ template.dump(cases, fname)
+
+ def prepare_task(self, test_name):
+ """Prepare resources for test run."""
+ inst_dir = getattr(config.CONF, 'dir_rally_inst')
+ try:
+ shutil.rmtree(os.path.join(inst_dir, 'plugins'))
+ shutil.rmtree(os.path.join(inst_dir, 'extra'))
+ except Exception: # pylint: disable=broad-except
+ pass
+ jobs_dir = os.path.join(
+ getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
+ shutil.copytree(os.path.join(jobs_dir, 'plugins'),
+ os.path.join(inst_dir, 'plugins'))
+ shutil.copytree(os.path.join(jobs_dir, 'extra'),
+ os.path.join(inst_dir, 'extra'))
+
+ task_name = self.task_yaml.get(test_name).get("task")
+ task = os.path.join(jobs_dir, task_name)
+ if not os.path.exists(task):
+ raise Exception("The scenario '%s' does not exist." % task)
+ LOGGER.debug('Scenario fetched from : %s', task)
+
+ if not os.path.exists(self.TEMP_DIR):
+ os.makedirs(self.TEMP_DIR)
+ task_file_name = os.path.join(self.TEMP_DIR, task_name)
+ self.apply_blacklist(task, task_file_name)
+ self.run_cmd = (["rally", "task", "start", "--task", task_file_name])
+ return True
diff --git a/functest/opnfv_tests/openstack/rally/rally_jobs.yaml b/functest/opnfv_tests/openstack/rally/rally_jobs.yaml
new file mode 100644
index 000000000..2092fb4cf
--- /dev/null
+++ b/functest/opnfv_tests/openstack/rally/rally_jobs.yaml
@@ -0,0 +1,3 @@
+---
+neutron:
+ task: task-neutron.yaml
diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index d89ebd634..22ddb1cc5 100644
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -71,7 +71,7 @@ class OSRallyTesting(unittest.TestCase):
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
- '_apply_blacklist')
+ 'apply_blacklist')
def test_prepare_test_list_missing_temp_dir(
self, mock_method, mock_os_makedirs, mock_path_exists):
mock_path_exists.side_effect = self.check_temp_dir
@@ -153,10 +153,13 @@ class OSRallyTesting(unittest.TestCase):
{'functions': ['no_migration'], 'tests': ['test']}]})
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'_migration_supported', return_value=False)
- def test_excl_func_default(self, mock_func, mock_yaml_load):
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
+ '_network_trunk_supported', return_value=False)
+ def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
self.assertEqual(self.rally_base.excl_func(), ['test'])
mock_func.assert_called()
+ mock_trunk.assert_called()
mock_yaml_load.assert_called()
@mock.patch('six.moves.builtins.open', side_effect=Exception)
@@ -263,8 +266,7 @@ class OSRallyTesting(unittest.TestCase):
with self.assertRaises(Exception):
self.rally_base.prepare_run()
- @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
- 'get_external_network')
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
def test_prepare_run_flavor_alt_creation_failed(self, *args):
# pylint: disable=unused-argument
self.rally_base.TESTS = ['test1', 'test2']
diff --git a/requirements.txt b/requirements.txt
index 66e388ca4..57e897178 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,3 +16,4 @@ Jinja2!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4,>=2.8 # BSD License (3 clause)
xtesting
os-client-config>=1.28.0 # Apache-2.0
shade>=1.17.0 # Apache-2.0
+ruamel.yaml.jinja2 # MIT
diff --git a/setup.cfg b/setup.cfg
index 4184957af..954dcaa9a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -43,6 +43,7 @@ xtesting.testcase =
heat_ims = functest.opnfv_tests.vnf.ims.heat_ims:HeatIms
vyos_vrouter = functest.opnfv_tests.vnf.router.cloudify_vrouter:CloudifyVrouter
juju_epc = functest.opnfv_tests.vnf.epc.juju_epc:JujuEpc
+ rally_jobs = functest.opnfv_tests.openstack.rally.rally:RallyJobs
[build_sphinx]
all_files = 1
diff --git a/tox.ini b/tox.ini
index ca78a0c2d..4f639d874 100644
--- a/tox.ini
+++ b/tox.ini
@@ -93,6 +93,7 @@ files =
ci
docker
functest/ci
+ functest/opnfv_tests/openstack/rally/rally_jobs.yaml
functest/opnfv_tests/vnf
commands =
yamllint -s {[testenv:yamllint]files}