Merge "Adding simple cmdline to run the Network service testcases"

2 files changed, 331 insertions, 0 deletions

diff --git a/tests/unit/cmd/test_NSBperf.py b/tests/unit/cmd/test_NSBperf.py
new file mode 100644
index 000000000..d63af4949
--- /dev/null
+++ b/tests/unit/cmd/test_NSBperf.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+import unittest
+import mock
+import subprocess
+import os
+
+from yardstick.cmd.NSBperf import YardstickNSCli
+from yardstick.cmd import NSBperf
+
+
+@mock.patch('__builtin__.raw_input', return_value='0')
+class TestHandler(unittest.TestCase):
+    def test_handler(self, test):
+        subprocess.call = mock.Mock(return_value=0)
+        self.assertRaises(SystemExit, NSBperf.handler)
+
+
+class TestYardstickNSCli(unittest.TestCase):
+    def test___init__(self):
+        yardstick_ns_cli = YardstickNSCli()
+        self.assertIsNotNone(yardstick_ns_cli)
+
+    def test_generate_final_report(self):
+        yardstick_ns_cli = YardstickNSCli()
+        test_case = "tc_baremetal_rfc2544_ipv4_1flow_1518B.yaml"
+        subprocess.call(["touch", "/tmp/yardstick.out"])
+        self.assertIsNone(yardstick_ns_cli.generate_final_report(test_case))
+
+    def test_generate_kpi_results(self):
+        yardstick_ns_cli = YardstickNSCli()
+        tkey = "cpu"
+        tgen = {"cpu": {"ipc": 0}}
+        self.assertIsNone(yardstick_ns_cli.generate_kpi_results(tkey, tgen))
+
+    def test_generate_nfvi_results(self):
+        yardstick_ns_cli = YardstickNSCli()
+        nfvi = {"collect_stats": {"cpu": {"ipc": 0, "Hz": 2.6}}}
+        self.assertIsNone(yardstick_ns_cli.generate_nfvi_results(nfvi))
+
+    def test_handle_list_options(self):
+        yardstick_ns_cli = YardstickNSCli()
+        CLI_PATH = os.path.dirname(os.path.realpath(__file__))
+        repo_dir = CLI_PATH + "/../../"
+        test_path = os.path.join(repo_dir, "../samples/vnf_samples/nsut/")
+        args = {"list_vnfs": True, "list": False}
+        self.assertRaises(SystemExit, yardstick_ns_cli.handle_list_options,
+                          args, test_path)
+        args = {"list_vnfs": False, "list": True}
+        self.assertRaises(SystemExit,
+                          yardstick_ns_cli.handle_list_options,
+                          args, test_path)
+
+    def test_main(self):
+        yardstick_ns_cli = YardstickNSCli()
+        yardstick_ns_cli.parse_arguments = mock.Mock(return_value=0)
+        yardstick_ns_cli.handle_list_options = mock.Mock(return_value=0)
+        yardstick_ns_cli.terminate_if_less_options = mock.Mock(return_value=0)
+        yardstick_ns_cli.run_test = mock.Mock(return_value=0)
+        self.assertIsNone(yardstick_ns_cli.main())
+
+    def test_parse_arguments(self):
+        yardstick_ns_cli = YardstickNSCli()
+        self.assertRaises(SystemExit, yardstick_ns_cli.parse_arguments)
+
+    def test_run_test(self):
+        cur_dir = os.getcwd()
+        CLI_PATH = os.path.dirname(os.path.realpath(__file__))
+        YARDSTICK_REPOS_DIR = os.path.join(CLI_PATH + "/../../")
+        test_path = os.path.join(YARDSTICK_REPOS_DIR,
+                                 "../samples/vnf_samples/nsut/")
+        yardstick_ns_cli = YardstickNSCli()
+        subprocess.check_output = mock.Mock(return_value=0)
+        args = {"vnf": "vpe",
+                "test": "tc_baremetal_rfc2544_ipv4_1flow_1518B.yaml"}
+        self.assertEqual(None, yardstick_ns_cli.run_test(args, test_path))
+        os.chdir(cur_dir)
+        args = {"vnf": "vpe1"}
+        self.assertEqual(None, yardstick_ns_cli.run_test(args, test_path))
+        os.chdir(cur_dir)
+        args = {"vnf": "vpe",
+                "test": "tc_baremetal_rfc2544_ipv4_1flow_1518B.yaml."}
+        self.assertEqual(None, yardstick_ns_cli.run_test(args, test_path))
+        os.chdir(cur_dir)
+        args = []
+        self.assertEqual(None, yardstick_ns_cli.run_test(args, test_path))
+        os.chdir(cur_dir)
+
+    def test_terminate_if_less_options(self):
+        yardstick_ns_cli = YardstickNSCli()
+        args = {"vnf": False}
+        self.assertRaises(SystemExit,
+                          yardstick_ns_cli.terminate_if_less_options, args)
+
+    def test_validate_input(self):
+        yardstick_ns_cli = YardstickNSCli()
+        self.assertEqual(1, yardstick_ns_cli.validate_input("", 4))
+        NSBperf.raw_input = lambda _: 'yes'
+        self.assertEqual(1, yardstick_ns_cli.validate_input(5, 4))
+        subprocess.call = mock.Mock(return_value=0)
+        self.assertEqual(0, yardstick_ns_cli.validate_input(2, 4))
+        subprocess.call = mock.Mock(return_value=0)
diff --git a/yardstick/cmd/NSBperf.py b/yardstick/cmd/NSBperf.py
new file mode 100755
index 000000000..7be44eeea
--- /dev/null
+++ b/yardstick/cmd/NSBperf.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""NSBPERF main script.
+"""
+
+from __future__ import absolute_import
+from __future__ import print_function
+import os
+import argparse
+import json
+import subprocess
+import signal
+
+CLI_PATH = os.path.dirname(os.path.realpath(__file__))
+REPO_PATH = os.path.abspath(os.path.join(CLI_PATH, os.pardir))
+PYTHONPATH = os.environ.get("PYTHONPATH", False)
+VIRTUAL_ENV = os.environ.get("VIRTUAL_ENV", False)
+
+if not PYTHONPATH or not VIRTUAL_ENV:
+    print("Please setup env PYTHONPATH & VIRTUAL_ENV environment varaible.")
+    raise SystemExit(1)
+
+
+def handler():
+    """ Capture ctrl+c and exit cli """
+    subprocess.call(["pkill", "-9", "yardstick"])
+    raise SystemExit(1)
+
+signal.signal(signal.SIGINT, handler)
+
+
+class YardstickNSCli(object):
+    """ This class handles yardstick network serivce testing """
+
+    def __init__(self):
+        super(YardstickNSCli, self).__init__()
+
+    @classmethod
+    def validate_input(cls, choice, choice_len):
+        """ Validate user inputs """
+        if not str(choice):
+            return 1
+
+        choice = int(choice)
+        if not 1 <= choice <= choice_len:
+            print("\nInvalid wrong choice...")
+            raw_input("Press Enter to continue...")
+            return 1
+        subprocess.call(['clear'])
+        return 0
+
+    @classmethod
+    def parse_arguments(cls):
+        """
+        Parse command line arguments.
+        """
+        parser = \
+            argparse.ArgumentParser(
+                prog=__file__,
+                formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        parser.add_argument('--version', action='version',
+                            version='%(prog)s 0.1')
+        parser.add_argument('--list', '--list-tests', action='store_true',
+                            help='list all tests and exit')
+        parser.add_argument('--list-vnfs', action='store_true',
+                            help='list all system vnfs and exit')
+
+        group = parser.add_argument_group('test selection options')
+        group.add_argument('--vnf', help='vnf to use')
+        group.add_argument('--test', help='test in use')
+
+        args = vars(parser.parse_args())
+
+        return args
+
+    @classmethod
+    def generate_kpi_results(cls, tkey, tgen):
+        """ Generate report for vnf & traffic generator kpis """
+        if tgen:
+            print("\n%s stats" % tkey)
+            print("----------------------------")
+            for key, value in tgen.items():
+                if key != "collect_stats":
+                    print(json.dumps({key: value}, indent=2))
+
+    @classmethod
+    def generate_nfvi_results(cls, nfvi):
+        """ Generate report for vnf & traffic generator kpis """
+        if nfvi:
+            nfvi_kpi = {k: v for k, v in nfvi.items() if k == 'collect_stats'}
+            if nfvi_kpi:
+                print("\nNFVi stats")
+                print("----------------------------")
+                for key, value in nfvi_kpi.items():
+                    print(json.dumps({key: value}, indent=2))
+
+    def generate_final_report(self, test_case):
+        """ Function will check if partial test results are available
+        and generates final report in rst format.
+"""
+
+        report_caption = '{}\n{} ({})\n{}\n\n'.format(
+            '================================================================',
+            'Performance report for',
+            os.path.splitext(test_case)[0].upper(),
+            '================================================================')
+        print(report_caption)
+        if os.path.isfile("/tmp/yardstick.out"):
+            lines = []
+            with open("/tmp/yardstick.out") as infile:
+                lines = infile.readlines()
+
+            if lines:
+                tc_res = json.loads(lines.pop(len(lines) - 1))
+                for key, value in tc_res["benchmark"]["data"].items():
+                    self.generate_kpi_results(key, value)
+                    self.generate_nfvi_results(value)
+
+    @classmethod
+    def handle_list_options(cls, args, test_path):
+        """ Process --list cli arguments if needed
+
+        :param args: A dictionary with all CLI arguments
+        """
+        if args['list_vnfs']:
+            vnfs = os.listdir(test_path)
+            print("VNF :")
+            print("================")
+            for index, vnf in enumerate(vnfs, 1):
+                print((' %-2s %s' % ('%s:' % str(index), vnf)))
+            raise SystemExit(0)
+
+        if args['list']:
+            vnfs = os.listdir(test_path)
+
+            print("Available Tests:")
+            print("*****************")
+            for vnf in vnfs:
+                testcases = os.listdir(test_path + vnf)
+                print(("VNF :(%s)" % vnf))
+                print("================")
+                for testcase in [tc for tc in testcases if "tc" in tc]:
+                    print('%s' % testcase)
+                print(os.linesep)
+            raise SystemExit(0)
+
+    @classmethod
+    def terminate_if_less_options(cls, args):
+        """ terminate cli if cmdline options is invalid """
+        if not (args["vnf"] and args["test"]):
+            print("CLI needs option, make sure to pass vnf, test")
+            print("eg: NSBperf.py --vnf <vnf untertest> --test <test yaml>")
+            raise SystemExit(1)
+
+    def run_test(self, args, test_path):
+        """ run requested test """
+        try:
+            vnf = args.get("vnf", "")
+            test = args.get("test", "")
+
+            vnf_dir = test_path + os.sep + vnf
+            if not os.path.exists(vnf_dir):
+                raise ValueError("'%s', vnf not supported." % vnf)
+
+            testcases = [tc for tc in os.listdir(vnf_dir) if "tc" in tc]
+            subtest = set([test]).issubset(testcases)
+            if not subtest:
+                raise ValueError("'%s', testcase not supported." % test)
+
+            os.chdir(vnf_dir)
+            # fixme: Use REST APIs to initiate testcases
+            subprocess.check_output(["yardstick", "--debug",
+                                     "task", "start", test])
+            self.generate_final_report(test)
+        except (IOError, ValueError):
+            print("Value/I/O error...")
+        except BaseException:
+            print("Test failed. Please verify test inputs & re-run the test..")
+            print("eg: NSBperf.py --vnf <vnf untertest> --test <test yaml>")
+
+    def main(self):
+        """Main function.
+        """
+        test_path = os.path.join(REPO_PATH, "../samples/vnf_samples/nsut/")
+        os.chdir(os.path.join(REPO_PATH, "../"))
+        args = self.parse_arguments()
+
+        # if required, handle list-* operations
+        self.handle_list_options(args, test_path)
+
+        # check for input params
+        self.terminate_if_less_options(args)
+
+        # run test
+        self.run_test(args, test_path)
+
+if __name__ == "__main__":
+    NS_CLI = YardstickNSCli()
+    NS_CLI.main()