Merge "add support for suite to support constraints and task_args"

8 files changed, 222 insertions, 46 deletions

diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml
index 6f2952f97..d5754b966 100644
--- a/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml
@@ -3,14 +3,16 @@
 # Measure latency between NFVI nodes using ping
 
 schema: "yardstick:task:0.1"
-
+{% set host = host or "node1.LF" %}
+{% set target = target or "node2.LF" %}
+{% set pod_info = pod_info or "etc/yardstick/nodes/compass_sclab_physical/pod.yaml" %}
 scenarios:
 -
   type: Ping
   options:
     packetsize: 100
-  host: node4.LF
-  target: node5.LF
+  host: {{host}}
+  target: {{target}}
 
   runner:
     type: Duration
@@ -25,5 +27,5 @@ scenarios:
 context:
   type: Node
   name: LF
-  file: etc/yardstick/nodes/compass_sclab_physical/pod.yaml
+  file: {{pod_info}}
 
diff --git a/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml b/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml
index d5f7bf824..c6dadac20 100644
--- a/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml
+++ b/tests/opnfv/test_suites/opnfv_os-nosdn-nofeature-ha_daily.yaml
@@ -7,16 +7,25 @@ name: "os-nosdn-nofeature-ha"
 test_cases_dir: "tests/opnfv/test_cases/"
 test_cases:
 -
-  file_name: opnfv_yardstick_tc002.yaml
+    file_name: opnfv_yardstick_tc002.yaml
 -
-  file_name: opnfv_yardstick_tc005.yaml
+    file_name: opnfv_yardstick_tc005.yaml
 -
-  file_name: opnfv_yardstick_tc010.yaml
+    file_name: opnfv_yardstick_tc010.yaml
 -
-  file_name: opnfv_yardstick_tc011.yaml
+    file_name: opnfv_yardstick_tc011.yaml
 -
-  file_name: opnfv_yardstick_tc012.yaml
+    file_name: opnfv_yardstick_tc012.yaml
 -
-  file_name: opnfv_yardstick_tc014.yaml
+    file_name: opnfv_yardstick_tc014.yaml
 -
-  file_name: opnfv_yardstick_tc037.yaml
+    file_name: opnfv_yardstick_tc037.yaml
+-
+    file_name: opnfv_yardstick_tc043.yaml
+    constraint:
+        installer: compass
+        pod: huawei-pod1
+    task_args:
+        huawei_pod1: '{"pod_info": "etc/yardstick/nodes/compass_sclab_physical/pod.yaml",
+        "host": "node4.LF","target": "node5.LF"}'
+
diff --git a/tests/unit/cmd/commands/no_constraint_no_args_scenario_sample.yaml b/tests/unit/cmd/commands/no_constraint_no_args_scenario_sample.yaml
new file mode 100644
index 000000000..4933b93ae
--- /dev/null
+++ b/tests/unit/cmd/commands/no_constraint_no_args_scenario_sample.yaml
@@ -0,0 +1,13 @@
+---
+# Huawei US bare daily task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "os-nosdn-nofeature-ha"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+    file_name: opnfv_yardstick_tc037.yaml
+-
+    file_name: opnfv_yardstick_tc043.yaml
+
diff --git a/tests/unit/cmd/commands/no_constraint_with_args_scenario_sample.yaml b/tests/unit/cmd/commands/no_constraint_with_args_scenario_sample.yaml
new file mode 100644
index 000000000..f39df7346
--- /dev/null
+++ b/tests/unit/cmd/commands/no_constraint_with_args_scenario_sample.yaml
@@ -0,0 +1,15 @@
+---
+# Huawei US bare daily task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "os-nosdn-nofeature-ha"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+    file_name: opnfv_yardstick_tc037.yaml
+-
+    file_name: opnfv_yardstick_tc043.yaml
+    task_args:
+        huawei-pod1: '{"host": "node1.LF","target": "node2.LF"}'
+
diff --git a/tests/unit/cmd/commands/test_task.py b/tests/unit/cmd/commands/test_task.py
index ccb0f2aec..0177fd08a 100644
--- a/tests/unit/cmd/commands/test_task.py
+++ b/tests/unit/cmd/commands/test_task.py
@@ -11,6 +11,7 @@
 
 # Unittest for yardstick.cmd.commands.task
 
+import os
 import mock
 import unittest
 
@@ -71,3 +72,79 @@ class TaskCommandsTestCase(unittest.TestCase):
         mock_os.environ.get.side_effect = ['compass', 'os-nosdn', 'huawei-pod1']
         result = t._check_precondition(cfg)
         self.assertTrue(result)
+
+    @mock.patch('yardstick.cmd.commands.task.os.environ')
+    def test_parse_suite_no_constraint_no_args(self, mock_environ):
+        SAMPLE_SCENARIO_PATH = "no_constraint_no_args_scenario_sample.yaml"
+        t = task.TaskParser(self._get_file_abspath(SAMPLE_SCENARIO_PATH))
+        mock_environ.get.side_effect = ['huawei-pod1', 'compass']
+        task_files, task_args, task_args_fnames = t.parse_suite()
+        print ("files=%s, args=%s, fnames=%s" % (task_files, task_args,
+               task_args_fnames))
+        self.assertEqual(task_files[0],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml')
+        self.assertEqual(task_files[1],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml')
+        self.assertEqual(task_args[0], None)
+        self.assertEqual(task_args[1], None)
+        self.assertEqual(task_args_fnames[0], None)
+        self.assertEqual(task_args_fnames[1], None)
+
+    @mock.patch('yardstick.cmd.commands.task.os.environ')
+    def test_parse_suite_no_constraint_with_args(self, mock_environ):
+        SAMPLE_SCENARIO_PATH = "no_constraint_with_args_scenario_sample.yaml"
+        t = task.TaskParser(self._get_file_abspath(SAMPLE_SCENARIO_PATH))
+        mock_environ.get.side_effect = ['huawei-pod1', 'compass']
+        task_files, task_args, task_args_fnames = t.parse_suite()
+        print ("files=%s, args=%s, fnames=%s" % (task_files, task_args,
+               task_args_fnames))
+        self.assertEqual(task_files[0],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml')
+        self.assertEqual(task_files[1],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml')
+        self.assertEqual(task_args[0], None)
+        self.assertEqual(task_args[1],
+                        '{"host": "node1.LF","target": "node2.LF"}')
+        self.assertEqual(task_args_fnames[0], None)
+        self.assertEqual(task_args_fnames[1], None)
+
+    @mock.patch('yardstick.cmd.commands.task.os.environ')
+    def test_parse_suite_with_constraint_no_args(self, mock_environ):
+        SAMPLE_SCENARIO_PATH = "with_constraint_no_args_scenario_sample.yaml"
+        t = task.TaskParser(self._get_file_abspath(SAMPLE_SCENARIO_PATH))
+        mock_environ.get.side_effect = ['huawei-pod1', 'compass']
+        task_files, task_args, task_args_fnames = t.parse_suite()
+        print ("files=%s, args=%s, fnames=%s" % (task_files, task_args,
+               task_args_fnames))
+        self.assertEqual(task_files[0],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml')
+        self.assertEqual(task_files[1],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml')
+        self.assertEqual(task_args[0], None)
+        self.assertEqual(task_args[1], None)
+        self.assertEqual(task_args_fnames[0], None)
+        self.assertEqual(task_args_fnames[1], None)
+
+    @mock.patch('yardstick.cmd.commands.task.os.environ')
+    def test_parse_suite_with_constraint_with_args(self, mock_environ):
+        SAMPLE_SCENARIO_PATH = "with_constraint_with_args_scenario_sample.yaml"
+        t = task.TaskParser(self._get_file_abspath(SAMPLE_SCENARIO_PATH))
+        mock_environ.get.side_effect = ['huawei-pod1', 'compass']
+        task_files, task_args, task_args_fnames = t.parse_suite()
+        print ("files=%s, args=%s, fnames=%s" % (task_files, task_args,
+               task_args_fnames))
+        self.assertEqual(task_files[0],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc037.yaml')
+        self.assertEqual(task_files[1],
+            'tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml')
+        self.assertEqual(task_args[0], None)
+        self.assertEqual(task_args[1],
+                        '{"host": "node1.LF","target": "node2.LF"}')
+        self.assertEqual(task_args_fnames[0], None)
+        self.assertEqual(task_args_fnames[1], None)
+
+    def _get_file_abspath(self, filename):
+        curr_path = os.path.dirname(os.path.abspath(__file__))
+        file_path = os.path.join(curr_path, filename)
+        return file_path
+
diff --git a/tests/unit/cmd/commands/with_constraint_no_args_scenario_sample.yaml b/tests/unit/cmd/commands/with_constraint_no_args_scenario_sample.yaml
new file mode 100644
index 000000000..8194a2361
--- /dev/null
+++ b/tests/unit/cmd/commands/with_constraint_no_args_scenario_sample.yaml
@@ -0,0 +1,16 @@
+---
+# Huawei US bare daily task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "os-nosdn-nofeature-ha"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+    file_name: opnfv_yardstick_tc037.yaml
+-
+    file_name: opnfv_yardstick_tc043.yaml
+    constraint:
+        installer: compass
+        pod: huawei-pod1
+
diff --git a/tests/unit/cmd/commands/with_constraint_with_args_scenario_sample.yaml b/tests/unit/cmd/commands/with_constraint_with_args_scenario_sample.yaml
new file mode 100644
index 000000000..86c9b2800
--- /dev/null
+++ b/tests/unit/cmd/commands/with_constraint_with_args_scenario_sample.yaml
@@ -0,0 +1,18 @@
+---
+# Huawei US bare daily task suite
+
+schema: "yardstick:suite:0.1"
+
+name: "os-nosdn-nofeature-ha"
+test_cases_dir: "tests/opnfv/test_cases/"
+test_cases:
+-
+    file_name: opnfv_yardstick_tc037.yaml
+-
+    file_name: opnfv_yardstick_tc043.yaml
+    constraint:
+        installer: compass
+        pod: huawei-pod1
+    task_args:
+        huawei-pod1: '{"host": "node1.LF","target": "node2.LF"}'
+
diff --git a/yardstick/cmd/commands/task.py b/yardstick/cmd/commands/task.py
index 587f62412..18b72e726 100644
--- a/yardstick/cmd/commands/task.py
+++ b/yardstick/cmd/commands/task.py
@@ -59,33 +59,29 @@ class TaskCommands(object):
         total_start_time = time.time()
         parser = TaskParser(args.inputfile[0])
 
-        suite_params = {}
         if args.suite:
-            suite_params = parser.parse_suite()
-            test_cases_dir = suite_params["test_cases_dir"]
-            if test_cases_dir[-1] != os.sep:
-                test_cases_dir += os.sep
-            task_files = [test_cases_dir + task
-                          for task in suite_params["task_fnames"]]
+            # 1.parse suite, return suite_params info
+            task_files, task_args, task_args_fnames = \
+                parser.parse_suite()
         else:
             task_files = [parser.path]
+            task_args = [args.task_args]
+            task_args_fnames = [args.task_args_file]
 
-        task_args = suite_params.get("task_args", [args.task_args])
-        task_args_fnames = suite_params.get("task_args_fnames",
-                                            [args.task_args_file])
+        LOG.info("\ntask_files:%s, \ntask_args:%s, \ntask_args_fnames:%s",
+                 task_files, task_args, task_args_fnames)
 
         if args.parse_only:
             sys.exit(0)
 
         if os.path.isfile(args.output_file):
             os.remove(args.output_file)
-
+        # parse task_files
         for i in range(0, len(task_files)):
             one_task_start_time = time.time()
             parser.path = task_files[i]
-            task_name = os.path.splitext(os.path.basename(task_files[i]))[0]
             scenarios, run_in_parallel, meet_precondition = parser.parse_task(
-                task_name, task_args[i], task_args_fnames[i])
+                 task_args[i], task_args_fnames[i])
 
             if not meet_precondition:
                 LOG.info("meet_precondition is %s, please check envrionment",
@@ -167,10 +163,34 @@ class TaskParser(object):
     def __init__(self, path):
         self.path = path
 
+    def _meet_constraint(self, task, cur_pod, cur_installer):
+        if "constraint" in task:
+            constraint = task.get('constraint', None)
+            if constraint is not None:
+                tc_fit_pod = constraint.get('pod', None)
+                tc_fit_installer = constraint.get('installer', None)
+                LOG.info("cur_pod:%s, cur_installer:%s,tc_constraints:%s",
+                         cur_pod, cur_installer, constraint)
+                if cur_pod and tc_fit_pod and cur_pod not in tc_fit_pod:
+                    return False
+                if cur_installer and tc_fit_installer and \
+                        cur_installer not in tc_fit_installer:
+                    return False
+        return True
+
+    def _get_task_para(self, task, cur_pod):
+        task_args = task.get('task_args', None)
+        if task_args is not None:
+            task_args = task_args.get(cur_pod, None)
+        task_args_fnames = task.get('task_args_fnames', None)
+        if task_args_fnames is not None:
+            task_args_fnames = task_args_fnames.get(cur_pod, None)
+        return task_args, task_args_fnames
+
     def parse_suite(self):
         '''parse the suite file and return a list of task config file paths
            and lists of optional parameters if present'''
-        print "Parsing suite file:", self.path
+        LOG.info("\nParsing suite file:%s", self.path)
 
         try:
             with open(self.path) as stream:
@@ -179,35 +199,40 @@ class TaskParser(object):
             sys.exit(ioerror)
 
         self._check_schema(cfg["schema"], "suite")
-        print "Starting suite:", cfg["name"]
+        LOG.info("\nStarting scenario:%s", cfg["name"])
 
         test_cases_dir = cfg.get("test_cases_dir", test_cases_dir_default)
-        task_fnames = []
-        task_args = []
-        task_args_fnames = []
+        if test_cases_dir[-1] != os.sep:
+            test_cases_dir += os.sep
+
+        cur_pod = os.environ.get('NODE_NAME', None)
+        cur_installer = os.environ.get('INSTALL_TYPE', None)
+
+        valid_task_files = []
+        valid_task_args = []
+        valid_task_args_fnames = []
 
         for task in cfg["test_cases"]:
-            task_fnames.append(task["file_name"])
-            if "task_args" in task:
-                task_args.append(task["task_args"])
+            # 1.check file_name
+            if "file_name" in task:
+                task_fname = task.get('file_name', None)
+                if task_fname is None:
+                    continue
             else:
-                task_args.append(None)
-
-            if "task_args_file" in task:
-                task_args_fnames.append(task["task_args_file"])
+                continue
+            # 2.check constraint
+            if self._meet_constraint(task, cur_pod, cur_installer):
+                valid_task_files.append(test_cases_dir + task_fname)
             else:
-                task_args_fnames.append(None)
-
-        suite_params = {
-            "test_cases_dir": test_cases_dir,
-            "task_fnames": task_fnames,
-            "task_args": task_args,
-            "task_args_fnames": task_args_fnames
-        }
+                continue
+            # 3.fetch task parameters
+            task_args, task_args_fnames = self._get_task_para(task, cur_pod)
+            valid_task_args.append(task_args)
+            valid_task_args_fnames.append(task_args_fnames)
 
-        return suite_params
+        return valid_task_files, valid_task_args, valid_task_args_fnames
 
-    def parse_task(self, task_name, task_args=None, task_args_file=None):
+    def parse_task(self, task_args=None, task_args_file=None):
         '''parses the task file and return an context and scenario instances'''
         print "Parsing task config:", self.path
 
@@ -262,6 +287,7 @@ class TaskParser(object):
         # add tc and task id for influxdb extended tags
         task_id = str(uuid.uuid4())
         for scenario in cfg["scenarios"]:
+            task_name = os.path.splitext(os.path.basename(self.path))[0]
             scenario["tc"] = task_name
             scenario["task_id"] = task_id