Add resiliency test caes into proposed_tests

JIRA: DOVETAIL-474

Add Bottlenecks resiliency test case "posca_factor_ping" into proposed_tests.

Change-Id: I76aea651014fb4722ddae07559f417d3f840eee2
Signed-off-by: xudan <xudan16@huawei.com>

11 files changed, 193 insertions, 13 deletions

diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 018edba0..b148df68 100644
--- a/dovetail/compliance/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -48,3 +48,5 @@ proposed_tests:
     - dovetail.sdnvpn.tc002
     - dovetail.sdnvpn.tc004
     - dovetail.sdnvpn.tc008
+    # resiliency
+    - dovetail.resiliency.tc001
diff --git a/dovetail/conf/bottlenecks_config.yml b/dovetail/conf/bottlenecks_config.yml
new file mode 100644
index 00000000..43df8c55
--- /dev/null
+++ b/dovetail/conf/bottlenecks_config.yml
@@ -0,0 +1,20 @@
+---
+bottlenecks:
+  image_name: opnfv/bottlenecks
+  docker_tag: cvp.0.4.0
+  opts: '-id --privileged=true'
+  config:
+    dir: '/home/opnfv/userconfig'
+  pre_condition:
+    - 'echo test for precondition in bottlenecks'
+  cmds:
+    - '/home/opnfv/bottlenecks/run_tests.sh -c {{validate_testcase}}'
+  post_condition:
+    - 'echo test for postcondition in bottlenecks'
+  result:
+    dir: '/tmp'
+    file_path: 'bottlenecks.log'
+  openrc: '/tmp/admin_rc.sh'
+  extra_container:
+    - 'Bottlenecks-Yardstick'
+    - 'Bottlenecks-ELK'
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index a5b262d1..d862a910 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -21,14 +21,21 @@ cli:
           - '-y'
         path:
           - 'yardstick/docker_tag'
-        help: 'Overwrite tag for yardstick docker container (e.g. stable or latest)'
+        help: 'Overwrite tag for yardstick docker container (e.g. danube.3.1)'
       func_tag:
         flags:
           - '--func_tag'
           - '-f'
         path:
           - 'functest/docker_tag'
-        help: 'Overwrite tag for functest docker container (e.g. stable or latest)'
+        help: 'Overwrite tag for functest docker container (e.g. cvp.0.2.0)'
+      bott_tag:
+        flags:
+          - '--bott_tag'
+          - '-b'
+        path:
+          - 'bottlenecks/docker_tag'
+        help: 'Overwrite tag for bottlenecks docker container (e.g. cvp.0.4.0)'
     control:
       testsuite:
         flags:
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index 36c8e5c1..98f9e814 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -32,6 +32,7 @@ testarea_supported:
   - ipv6
   - sdnvpn
   - vping
+  - resiliency
 
 functest_testsuite:
   - refstack_defcore
@@ -66,10 +67,12 @@ parameters:
 include_config:
   - functest_config.yml
   - yardstick_config.yml
+  - bottlenecks_config.yml
 
 test_project:
   - 'yardstick'
   - 'functest'
+  - 'bottlenecks'
 
 validate_input:
   valid_docker_tag:
@@ -82,3 +85,4 @@ validate_input:
     - 'cvp.0.1.0'
     - 'cvp.0.2.0'
     - 'cvp.0.3.0'
+    - 'cvp.0.4.0'
diff --git a/dovetail/container.py b/dovetail/container.py
index 1a5867a7..ede50a80 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -18,7 +18,8 @@ from utils.dovetail_config import DovetailConfig as dt_cfg
 class Container(object):
 
     container_list = {}
-    has_pull_latest_image = {'yardstick': False, 'functest': False}
+    has_pull_latest_image = {'yardstick': False, 'functest': False,
+                             'bottlenecks': False}
 
     logger = None
 
@@ -123,6 +124,19 @@ class Container(object):
         return "%s %s %s" % (envs, log_vol, key_vol)
 
     @classmethod
+    def set_bottlenecks_config(cls, testcase_name):
+        dovetail_config = dt_cfg.dovetail_config
+        yard_tag = dovetail_config['yardstick']['docker_tag']
+        docker_vol = '-v /var/run/docker.sock:/var/run/docker.sock'
+        env = ('-e Yardstick_TAG={} -e OUTPUT_FILE={}.out'
+               .format(yard_tag, testcase_name))
+        report = ""
+        if dovetail_config['report_dest'].startswith("http"):
+            report = ("-e BOTTLENECKS_DB_TARGET={}"
+                      .format(dovetail_config['report_dest']))
+        return "{} {} {}".format(docker_vol, env, report)
+
+    @classmethod
     def create(cls, type, testcase_name):
         dovetail_config = dt_cfg.dovetail_config
         docker_image = cls.get_docker_image(type)
@@ -144,6 +158,8 @@ class Container(object):
             config = cls.set_functest_config(testcase_name)
         if type.lower() == "yardstick":
             config = cls.set_yardstick_config()
+        if type.lower() == "bottlenecks":
+            config = cls.set_bottlenecks_config(testcase_name)
         if not config:
             return None
 
@@ -284,11 +300,23 @@ class Container(object):
         return image_id
 
     @classmethod
-    def clean(cls, container_id):
-        cmd1 = 'sudo docker stop %s' % (container_id)
-        dt_utils.exec_cmd(cmd1, cls.logger)
-        cmd2 = 'sudo docker rm %s' % (container_id)
-        dt_utils.exec_cmd(cmd2, cls.logger)
+    def check_container_exist(cls, container_name):
+        cmd = ('sudo docker ps -aq -f name={}'.format(container_name))
+        ret, msg = dt_utils.exec_cmd(cmd, cls.logger)
+        if ret == 0 and msg:
+            return True
+        return False
+
+    @classmethod
+    def clean(cls, container_id, valid_type):
+        cmd = ('sudo docker rm -f {}'.format(container_id))
+        dt_utils.exec_cmd(cmd, cls.logger)
+        if valid_type.lower() == 'bottlenecks':
+            containers = dt_cfg.dovetail_config[valid_type]['extra_container']
+            for container in containers:
+                if cls.check_container_exist(container):
+                    cmd = ('sudo docker rm -f {}'.format(container))
+                    dt_utils.exec_cmd(cmd, cls.logger)
 
     @classmethod
     def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
diff --git a/dovetail/report.py b/dovetail/report.py
index 08780b88..4743c256 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -25,7 +25,7 @@ from testcase import Testcase
 
 class Report(object):
 
-    results = {'functest': {}, 'yardstick': {}, 'shell': {}}
+    results = {'functest': {}, 'yardstick': {}, 'bottlenecks': {}, 'shell': {}}
 
     logger = None
 
@@ -339,6 +339,53 @@ class YardstickCrawler(object):
         return None
 
 
+class BottlenecksCrawler(object):
+
+    logger = None
+
+    def __init__(self):
+        self.type = 'bottlenecks'
+        self.logger.debug('Create crawler: {}'.format(self.type))
+
+    @classmethod
+    def create_log(cls):
+        cls.logger = \
+            dt_logger.Logger(__name__ + '.BottlenecksCrawler').getLogger()
+
+    def crawl(self, testcase=None):
+        report_dest = dt_cfg.dovetail_config['report_dest']
+        if report_dest.lower() == 'file':
+            return self.crawl_from_file(testcase)
+
+        if report_dest.lower().startswith('http'):
+            return self.crawl_from_url(testcase)
+
+    def crawl_from_file(self, testcase=None):
+        file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
+                                 testcase.name() + '.out')
+        if not os.path.exists(file_path):
+            self.logger.error('Result file not found: {}'.format(file_path))
+            return None
+        criteria = 'FAIL'
+        with open(file_path, 'r') as f:
+            for jsonfile in f:
+                data = json.loads(jsonfile)
+                try:
+                    if 'PASS' == data["data_body"]["result"]:
+                        criteria = 'PASS'
+                    else:
+                        criteria = 'FAIL'
+                        break
+                except KeyError as e:
+                    self.logger.exception('Pass flag not found {}'.format(e))
+        json_results = {'criteria': criteria}
+        self.logger.debug('Results: {}'.format(str(json_results)))
+        return json_results
+
+    def crawl_from_url(self, testcase=None):
+        return None
+
+
 class ShellCrawler(object):
 
     def __init__(self):
@@ -364,6 +411,7 @@ class CrawlerFactory(object):
 
     CRAWLER_MAP = {'functest': FunctestCrawler,
                    'yardstick': YardstickCrawler,
+                   'bottlenecks': BottlenecksCrawler,
                    'shell': ShellCrawler}
 
     @classmethod
@@ -466,6 +514,24 @@ class YardstickChecker(object):
         return
 
 
+class BottlenecksChecker(object):
+
+    logger = None
+
+    @classmethod
+    def create_log(cls):
+        cls.logger = \
+            dt_logger.Logger(__name__ + '.BottlenecksChecker').getLogger()
+
+    @staticmethod
+    def check(testcase, result):
+        if not result:
+            testcase.passed('FAIL')
+        else:
+            testcase.passed(result['criteria'])
+        return
+
+
 class ShellChecker(object):
 
     @staticmethod
@@ -480,6 +546,7 @@ class CheckerFactory(object):
 
     CHECKER_MAP = {'functest': FunctestChecker,
                    'yardstick': YardstickChecker,
+                   'bottlenecks': BottlenecksChecker,
                    'shell': ShellChecker}
 
     @classmethod
diff --git a/dovetail/run.py b/dovetail/run.py
index f89671f0..7515023b 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -22,8 +22,8 @@ from container import Container
 from testcase import Testcase
 from testcase import Testsuite
 from report import Report
-from report import FunctestCrawler, YardstickCrawler
-from report import FunctestChecker, YardstickChecker
+from report import FunctestCrawler, YardstickCrawler, BottlenecksCrawler
+from report import FunctestChecker, YardstickChecker, BottlenecksChecker
 from utils.dovetail_config import DovetailConfig as dt_cfg
 from test_runner import DockerRunner, ShellRunner
 
@@ -90,6 +90,8 @@ def check_tc_result(testcase, logger):
             result_file = os.path.join(result_dir, testcase.name() + '.out')
         elif validate_type.lower() == 'functest':
             result_file = os.path.join(result_dir, functest_result)
+        elif validate_type.lower() == 'bottlenecks':
+            result_file = os.path.join(result_dir, testcase.name() + '.out')
         else:
             logger.error("Don't support {} now.".format(validate_type))
             return
@@ -107,6 +109,7 @@ def validate_input(input_dict, check_dict, logger):
     # for 'func_tag' and 'yard_tag' options
     func_tag = input_dict['func_tag']
     yard_tag = input_dict['yard_tag']
+    bott_tag = input_dict['bott_tag']
     valid_tag = check_dict['valid_docker_tag']
     if func_tag is not None and func_tag not in valid_tag:
         logger.error("The input option 'func_tag' can't be {}, "
@@ -116,6 +119,10 @@ def validate_input(input_dict, check_dict, logger):
         logger.error("The input option 'yard_tag' can't be {}, "
                      "valid values are {}.".format(yard_tag, valid_tag))
         raise SystemExit(1)
+    if bott_tag is not None and bott_tag not in valid_tag:
+        logger.error("The input option 'bott_tag' can't be {}, "
+                     "valid values are {}.".format(bott_tag, valid_tag))
+        raise SystemExit(1)
 
     # for 'report' option
     report = input_dict['report']
@@ -164,8 +171,10 @@ def create_logs():
     Report.create_log()
     FunctestCrawler.create_log()
     YardstickCrawler.create_log()
+    BottlenecksCrawler.create_log()
     FunctestChecker.create_log()
     YardstickChecker.create_log()
+    BottlenecksChecker.create_log()
     Testcase.create_log()
     Testsuite.create_log()
     DockerRunner.create_log()
@@ -246,6 +255,7 @@ def main(*args, **kwargs):
         if(kwargs['report'].endswith('/')):
             kwargs['report'] = kwargs['report'][0:kwargs['report'].rfind('/')]
         dt_cfg.dovetail_config['report_dest'] = kwargs['report']
+        dt_cfg.update_cmds()
 
     if kwargs['offline']:
         dt_cfg.dovetail_config['offline'] = True
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index b3fd7a3f..603156fe 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -111,7 +111,7 @@ class DockerRunner(object):
                 ret, msg = Container.exec_cmd(container_id, cmd)
         self.testcase.cleaned(True)
 
-        Container.clean(container_id)
+        Container.clean(container_id, self.type)
 
 
 class FunctestRunner(DockerRunner):
@@ -128,6 +128,13 @@ class YardstickRunner(DockerRunner):
         super(YardstickRunner, self).__init__(testcase)
 
 
+class BottlenecksRunner(DockerRunner):
+
+    def __init__(self, testcase):
+        self.type = 'bottlenecks'
+        super(BottlenecksRunner, self).__init__(testcase)
+
+
 class ShellRunner(object):
 
     logger = None
@@ -190,6 +197,7 @@ class TestRunnerFactory(object):
     TEST_RUNNER_MAP = {
         "functest": FunctestRunner,
         "yardstick": YardstickRunner,
+        "bottlenecks": BottlenecksRunner,
         "shell": ShellRunner,
     }
 
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 7b012c88..bdfd3d35 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -254,13 +254,30 @@ class FunctestTestcase(Testcase):
 
 class YardstickTestcase(Testcase):
 
-    validate_testcae_list = {}
+    validate_testcase_list = {}
 
     def __init__(self, testcase_yaml):
         super(YardstickTestcase, self).__init__(testcase_yaml)
         self.type = 'yardstick'
 
 
+class BottlenecksTestcase(Testcase):
+
+    validate_testcase_list = {}
+
+    def __init__(self, testcase_yaml):
+        super(BottlenecksTestcase, self).__init__(testcase_yaml)
+        self.type = 'bottlenecks'
+        self._update_cmds()
+
+    def _update_cmds(self):
+        if dt_cfg.dovetail_config['report_dest'].startswith("http"):
+            try:
+                self.testcase['validate']['cmds'][0] += ' --report'
+            except KeyError:
+                return
+
+
 class ShellTestcase(Testcase):
 
     validate_testcase_list = {}
@@ -274,6 +291,7 @@ class TestcaseFactory(object):
     TESTCASE_TYPE_MAP = {
         'functest': FunctestTestcase,
         'yardstick': YardstickTestcase,
+        'bottlenecks': BottlenecksTestcase,
         'shell': ShellTestcase,
     }
 
diff --git a/dovetail/testcase/resiliency.tc001.yml b/dovetail/testcase/resiliency.tc001.yml
new file mode 100644
index 00000000..86399849
--- /dev/null
+++ b/dovetail/testcase/resiliency.tc001.yml
@@ -0,0 +1,11 @@
+---
+dovetail.resiliency.tc001:
+  name: dovetail.resiliency.tc001
+  objective: > # This test case verifies the ability of the SUT setting up VM pairs
+    # for different tenants and providing acceptable capacity after the amount of
+    # VM pairs reaches certain quantity.
+  validate:
+    type: bottlenecks
+    testcase: posca_factor_ping
+  report:
+    sub_testcase_list:
diff --git a/dovetail/utils/dovetail_config.py b/dovetail/utils/dovetail_config.py
index f8193e5d..5e100d69 100644
--- a/dovetail/utils/dovetail_config.py
+++ b/dovetail/utils/dovetail_config.py
@@ -54,3 +54,8 @@ class DovetailConfig(object):
     def update_non_envs(cls, path, value):
         if value:
             cls.set_leaf_dict(cls.dovetail_config, path, value)
+
+    @classmethod
+    def update_cmds(cls):
+        if cls.dovetail_config['report_dest'].startswith("http"):
+            cls.dovetail_config['bottlenecks']['cmds'][0] += ' --report'