Add support for Rally OpenStack CI test cases

Include Neutron rally-jobs.

Change-Id: Ic1122962bd7d876d179729640a8a321bc36442d8
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>

13 files changed, 170 insertions, 17 deletions

diff --git a/ci/run.yaml b/ci/run.yaml
index 9cc374c0c..385d38fb1 100644
--- a/ci/run.yaml
+++ b/ci/run.yaml
@@ -66,6 +66,7 @@
       - tempest_smoke
       - neutron-tempest-plugin-api
       - rally_sanity
+      - rally_jobs
       - refstack_defcore
       - patrole
       - snaps_smoke
@@ -86,6 +87,7 @@
             - name: '{repo}-functest-smoke-tempest_smoke-{tag}-run'
             - name: '{repo}-functest-smoke-neutron-tempest-plugin-api-{tag}-run'
             - name: '{repo}-functest-smoke-rally_sanity-{tag}-run'
+            - name: '{repo}-functest-smoke-rally_jobs-{tag}-run'
             - name: '{repo}-functest-smoke-refstack_defcore-{tag}-run'
             - name: '{repo}-functest-smoke-patrole-{tag}-run'
             - name: '{repo}-functest-smoke-snaps_smoke-{tag}-run'
diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile
index e222fe8fc..19fa9d9c1 100644
--- a/docker/smoke/Dockerfile
+++ b/docker/smoke/Dockerfile
@@ -6,6 +6,7 @@ ARG REFSTACK_TARGET=2018.02
 ARG PATROLE_TAG=0.4.0
 ARG NEUTRON_TAG=master
 ARG BARBICAN_TAG=0.1.0
+ARG NEUTRON_RALLY_TAG=master
 
 RUN apk --no-cache add --virtual .build-deps --update \
         python-dev build-base linux-headers libffi-dev \
@@ -41,6 +42,11 @@ RUN apk --no-cache add --virtual .build-deps --update \
     mkdir -p /etc/neutron /etc/glance && \
     wget -q -O /etc/neutron/policy.json https://git.openstack.org/cgit/openstack/neutron/plain/etc/policy.json?h=$OPENSTACK_TAG && \
     wget -q -O /etc/glance/policy.json https://git.openstack.org/cgit/openstack/glance/plain/etc/policy.json?h=$OPENSTACK_TAG && \
+    git clone --depth 1 -b $NEUTRON_RALLY_TAG https://git.openstack.org/openstack/neutron.git /src/neutron && \
+    (cd /src/neutron && git fetch --tags origin $NEUTRON_RALLY_TAG && git checkout FETCH_HEAD) && \
+    mkdir -p /home/opnfv/functest/data/rally/neutron && \
+    cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
+    rm -r /src/neutron && \
     apk del .build-deps
 COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
 CMD ["run_tests", "-t", "all"]
diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 254c39b27..55f7a102a 100644
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -53,6 +53,17 @@ tiers:
                     name: rally_sanity
 
             -
+                case_name: rally_jobs
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs a group of Rally jobs used in
+                    OpenStack gating
+                run:
+                    name: rally_jobs
+
+            -
                 case_name: refstack_defcore
                 project_name: functest
                 criteria: 100
diff --git a/docs/release/release-notes/functest-release.rst b/docs/release/release-notes/functest-release.rst
index ebb99247f..67fa3dcbb 100644
--- a/docs/release/release-notes/functest-release.rst
+++ b/docs/release/release-notes/functest-release.rst
@@ -131,6 +131,7 @@ New test cases
  * singlevm2
  * cinder_test
  * neutron-tempest-plugin-api
+ * rally_jobs
  * networking-bgpvpn
  * networking-sfc
  * barbican
diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml
index 557a67b94..dbea320c9 100644
--- a/functest/ci/config_functest.yaml
+++ b/functest/ci/config_functest.yaml
@@ -11,6 +11,7 @@ general:
         functest_conf: /home/opnfv/functest/conf
         functest_data: /home/opnfv/functest/data
         ims_data: /home/opnfv/functest/data/ims/
+        rally_data: /home/opnfv/functest/data/rally
         refstack_data: /home/opnfv/functest/data/refstack
         router_data: /home/opnfv/functest/data/router/opnfv-vnf-data
         functest_images: /home/opnfv/functest/images
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index e11248f90..16365b0b7 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -225,6 +225,17 @@ tiers:
                     name: rally_sanity
 
             -
+                case_name: rally_jobs
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs a group of Rally jobs used in
+                    OpenStack gating
+                run:
+                    name: rally_jobs
+
+            -
                 case_name: refstack_defcore
                 project_name: functest
                 criteria: 100
diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt
index 0900f21c1..1ade43eaf 100644
--- a/functest/opnfv_tests/openstack/rally/blacklist.txt
+++ b/functest/opnfv_tests/openstack/rally/blacklist.txt
@@ -17,3 +17,8 @@ functionality:
             - NovaServers.boot_server_attach_created_volume_and_live_migrate
             - NovaServers.boot_server_from_volume_and_live_migrate
             - NovaServers.boot_and_migrate_server
+    -
+        functions:
+            - no_net_trunk_service
+        tests:
+            - '^NeutronTrunk'
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index 00e759150..0f74896d7 100644
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -16,11 +16,13 @@ import json
 import logging
 import os
 import re
+import shutil
 import subprocess
 import time
 
 import pkg_resources
 import prettytable
+from ruamel.yaml import YAML
 from six.moves import configparser
 from xtesting.core import testcase
 from xtesting.energy import energy
@@ -53,7 +55,6 @@ class RallyBase(singlevm.VmReady2):
     TENANTS_AMOUNT = 3
     ITERATIONS_AMOUNT = 10
     CONCURRENCY = 4
-    RESULTS_DIR = os.path.join(getattr(config.CONF, 'dir_results'), 'rally')
     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
     TEMP_DIR = os.path.join(RALLY_DIR, "var")
 
@@ -75,6 +76,9 @@ class RallyBase(singlevm.VmReady2):
             role_name, user=self.project.user.id,
             project=self.project.project.id,
             domain=self.project.domain.id)
+        self.results_dir = os.path.join(
+            getattr(config.CONF, 'dir_results'), self.case_name)
+        self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
         self.creators = []
         self.summary = []
         self.scenario_dir = ''
@@ -86,8 +90,8 @@ class RallyBase(singlevm.VmReady2):
         self.compute_cnt = 0
         self.flavor_alt = None
         self.tests = []
-        self.task_file = ''
         self.run_cmd = ''
+        self.network_extensions = []
 
     def _build_task_args(self, test_file_name):
         """Build arguments for the Rally task."""
@@ -138,7 +142,7 @@ class RallyBase(singlevm.VmReady2):
         if not os.path.exists(self.TEMP_DIR):
             os.makedirs(self.TEMP_DIR)
 
-        self._apply_blacklist(scenario_file_name, test_file_name)
+        self.apply_blacklist(scenario_file_name, test_file_name)
         return test_file_name
 
     @staticmethod
@@ -204,7 +208,12 @@ class RallyBase(singlevm.VmReady2):
         """Determine if migration is supported."""
         if self.compute_cnt > 1:
             return True
+        return False
 
+    def _network_trunk_supported(self):
+        """Determine if network trunk service is available"""
+        if 'trunk' in self.network_extensions:
+            return True
         return False
 
     @staticmethod
@@ -261,6 +270,8 @@ class RallyBase(singlevm.VmReady2):
 
             if not self._migration_supported():
                 func_list.append("no_migration")
+            if not self._network_trunk_supported():
+                func_list.append("no_net_trunk_service")
 
             if 'functionality' in black_list_yaml.keys():
                 for item in black_list_yaml['functionality']:
@@ -274,7 +285,7 @@ class RallyBase(singlevm.VmReady2):
 
         return black_tests
 
-    def _apply_blacklist(self, case_file_name, result_file_name):
+    def apply_blacklist(self, case_file_name, result_file_name):
         """Apply blacklist."""
         LOGGER.debug("Applying blacklist...")
         cases_file = open(case_file_name, 'r')
@@ -317,10 +328,10 @@ class RallyBase(singlevm.VmReady2):
     def _save_results(self, test_name, task_id):
         """ Generate and save task execution results"""
         # check for result directory and create it otherwise
-        if not os.path.exists(self.RESULTS_DIR):
+        if not os.path.exists(self.results_dir):
             LOGGER.debug('%s does not exist, we create it.',
-                         self.RESULTS_DIR)
-            os.makedirs(self.RESULTS_DIR)
+                         self.results_dir)
+            os.makedirs(self.results_dir)
 
         # put detailed result to log
         cmd = (["rally", "task", "detailed", "--uuid", task_id])
@@ -329,8 +340,8 @@ class RallyBase(singlevm.VmReady2):
         LOGGER.info("%s\n%s", " ".join(cmd), output)
 
         # save report as JSON
-        report_json_name = 'opnfv-{}.json'.format(test_name)
-        report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
+        report_json_name = '{}.json'.format(test_name)
+        report_json_dir = os.path.join(self.results_dir, report_json_name)
         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
                 "--out", report_json_dir])
         LOGGER.debug('running command: %s', cmd)
@@ -338,8 +349,8 @@ class RallyBase(singlevm.VmReady2):
         LOGGER.info("%s\n%s", " ".join(cmd), output)
 
         # save report as HTML
-        report_html_name = 'opnfv-{}.html'.format(test_name)
-        report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
+        report_html_name = '{}.html'.format(test_name)
+        report_html_dir = os.path.join(self.results_dir, report_html_name)
         cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
                 "--out", report_html_dir])
         LOGGER.debug('running command: %s', cmd)
@@ -409,7 +420,6 @@ class RallyBase(singlevm.VmReady2):
         else:
             raise Exception("Test name '%s' is invalid" % self.test_name)
 
-        self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
         if not os.path.exists(self.task_file):
             LOGGER.error("Task file '%s' does not exist.", self.task_file)
             raise Exception("Task file '{}' does not exist.".
@@ -417,6 +427,7 @@ class RallyBase(singlevm.VmReady2):
 
         self.update_keystone_default_role()
         self.compute_cnt = len(self.cloud.list_hypervisors())
+        self.network_extensions = self.cloud.get_network_extensions()
         self.flavor_alt = self.create_flavor_alt()
         LOGGER.debug("flavor: %s", self.flavor_alt)
 
@@ -464,7 +475,7 @@ class RallyBase(singlevm.VmReady2):
             except ZeroDivisionError:
                 success_avg = 0
             success_str = str("{:0.2f}".format(success_avg)) + '%'
-            duration_str = time.strftime("%M:%S",
+            duration_str = time.strftime("%H:%M:%S",
                                          time.gmtime(item['overall_duration']))
             res_table.add_row([item['test_name'], duration_str,
                                item['nb_tests'], success_str])
@@ -559,3 +570,100 @@ class RallyFull(RallyBase):
         self.test_name = 'all'
         self.smoke = False
         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
+
+
+class RallyJobs(RallyBase):
+    """Rally OpenStack CI testcase implementation."""
+
+    TESTS = ["neutron"]
+
+    def __init__(self, **kwargs):
+        """Initialize RallyJobs object."""
+        if "case_name" not in kwargs:
+            kwargs["case_name"] = "rally_jobs"
+        super(RallyJobs, self).__init__(**kwargs)
+        self.test_name = 'all'
+        self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
+        self.task_yaml = None
+
+    def prepare_run(self):
+        """Create resources needed by test scenarios."""
+        super(RallyJobs, self).prepare_run()
+        with open(os.path.join(self.RALLY_DIR,
+                               'rally_jobs.yaml'), 'r') as task_file:
+            self.task_yaml = yaml.safe_load(task_file)
+
+        if not all(task in self.task_yaml for task in self.tests):
+            raise Exception("Test '%s' not in '%s'" %
+                            (self.test_name, self.tests))
+
+    def apply_blacklist(self, case_file_name, result_file_name):
+        # pylint: disable=too-many-branches
+        """Apply blacklist."""
+        LOGGER.debug("Applying blacklist...")
+        black_tests = list(set(self.excl_func() +
+                               self.excl_scenario()))
+        if black_tests:
+            LOGGER.debug("Blacklisted tests: %s", str(black_tests))
+
+        template = YAML(typ='jinja2')
+        with open(case_file_name, 'r') as fname:
+            cases = template.load(fname)
+        if cases.get("version", 1) == 1:
+            # scenarios in dictionary
+            for name in cases.keys():
+                if self.in_iterable_re(name, black_tests):
+                    cases.pop(name)
+        else:
+            # workloads in subtasks
+            for sind, subtask in enumerate(cases.get('subtasks', [])):
+                idx = []
+                for wind, workload in enumerate(subtask.get('workloads', [])):
+                    scenario = workload.get('scenario', {})
+                    for name in scenario.keys():
+                        if self.in_iterable_re(name, black_tests):
+                            idx.append(wind)
+                            break
+                for wind in reversed(idx):
+                    cases['subtasks'][sind]['workloads'].pop(wind)
+            # scenarios in subtasks
+            idx = []
+            for sind, subtask in enumerate(cases.get('subtasks', [])):
+                scenario = subtask.get('scenario', {})
+                for name in scenario.keys():
+                    if self.in_iterable_re(name, black_tests):
+                        idx.append(sind)
+                        break
+            for sind in reversed(idx):
+                cases['subtasks'].pop(sind)
+
+        with open(result_file_name, 'w') as fname:
+            template.dump(cases, fname)
+
+    def prepare_task(self, test_name):
+        """Prepare resources for test run."""
+        inst_dir = getattr(config.CONF, 'dir_rally_inst')
+        try:
+            shutil.rmtree(os.path.join(inst_dir, 'plugins'))
+            shutil.rmtree(os.path.join(inst_dir, 'extra'))
+        except Exception:  # pylint: disable=broad-except
+            pass
+        jobs_dir = os.path.join(
+            getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
+        shutil.copytree(os.path.join(jobs_dir, 'plugins'),
+                        os.path.join(inst_dir, 'plugins'))
+        shutil.copytree(os.path.join(jobs_dir, 'extra'),
+                        os.path.join(inst_dir, 'extra'))
+
+        task_name = self.task_yaml.get(test_name).get("task")
+        task = os.path.join(jobs_dir, task_name)
+        if not os.path.exists(task):
+            raise Exception("The scenario '%s' does not exist." % task)
+        LOGGER.debug('Scenario fetched from : %s', task)
+
+        if not os.path.exists(self.TEMP_DIR):
+            os.makedirs(self.TEMP_DIR)
+        task_file_name = os.path.join(self.TEMP_DIR, task_name)
+        self.apply_blacklist(task, task_file_name)
+        self.run_cmd = (["rally", "task", "start", "--task", task_file_name])
+        return True
diff --git a/functest/opnfv_tests/openstack/rally/rally_jobs.yaml b/functest/opnfv_tests/openstack/rally/rally_jobs.yaml
new file mode 100644
index 000000000..2092fb4cf
--- /dev/null
+++ b/functest/opnfv_tests/openstack/rally/rally_jobs.yaml
@@ -0,0 +1,3 @@
+---
+neutron:
+  task: task-neutron.yaml
diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index d89ebd634..22ddb1cc5 100644
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -71,7 +71,7 @@ class OSRallyTesting(unittest.TestCase):
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
-                '_apply_blacklist')
+                'apply_blacklist')
     def test_prepare_test_list_missing_temp_dir(
             self, mock_method, mock_os_makedirs, mock_path_exists):
         mock_path_exists.side_effect = self.check_temp_dir
@@ -153,10 +153,13 @@ class OSRallyTesting(unittest.TestCase):
                     {'functions': ['no_migration'], 'tests': ['test']}]})
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 '_migration_supported', return_value=False)
-    def test_excl_func_default(self, mock_func, mock_yaml_load):
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
+                '_network_trunk_supported', return_value=False)
+    def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
         self.assertEqual(self.rally_base.excl_func(), ['test'])
         mock_func.assert_called()
+        mock_trunk.assert_called()
         mock_yaml_load.assert_called()
 
     @mock.patch('six.moves.builtins.open', side_effect=Exception)
@@ -263,8 +266,7 @@ class OSRallyTesting(unittest.TestCase):
         with self.assertRaises(Exception):
             self.rally_base.prepare_run()
 
-    @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
-                'get_external_network')
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
     def test_prepare_run_flavor_alt_creation_failed(self, *args):
         # pylint: disable=unused-argument
         self.rally_base.TESTS = ['test1', 'test2']
diff --git a/requirements.txt b/requirements.txt
index 83bc9850b..298cc38ad 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,3 +16,4 @@ Jinja2 # BSD License (3 clause)
 xtesting
 os-client-config # Apache-2.0
 shade # Apache-2.0
+ruamel.yaml.jinja2 # MIT
diff --git a/setup.cfg b/setup.cfg
index 4184957af..954dcaa9a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -43,6 +43,7 @@ xtesting.testcase =
     heat_ims = functest.opnfv_tests.vnf.ims.heat_ims:HeatIms
     vyos_vrouter = functest.opnfv_tests.vnf.router.cloudify_vrouter:CloudifyVrouter
     juju_epc = functest.opnfv_tests.vnf.epc.juju_epc:JujuEpc
+    rally_jobs = functest.opnfv_tests.openstack.rally.rally:RallyJobs
 
 [build_sphinx]
 all_files = 1
diff --git a/tox.ini b/tox.ini
index 45f233e52..2b3e07ea9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -93,6 +93,7 @@ files =
   ci
   docker
   functest/ci
+  functest/opnfv_tests/openstack/rally/rally_jobs.yaml
   functest/opnfv_tests/vnf
 commands =
   yamllint -s {[testenv:yamllint]files}