[dovetail tool] support shell scripts for testcase validation

JIRA: DOVETAIL-46

1. for now a testcase has two kinds of validation types(functest, yardstick),
and it is not enough to check the complete funcionality
2. add new validation type(shell) for extra validation of the test case to make
result more accurate and more convincing.

Change-Id: I45dca6b8dbd888757da163189d261f6e4dba5034
Signed-off-by: Leo Wang <grakiss.wanglei@huawei.com>

1 files changed, 14 insertions, 11 deletions

diff --git a/dovetail/report.py b/dovetail/report.py
index 2e2a24f3..1f970b29 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -30,7 +30,7 @@ def get_pass_str(passed):
 
 class Report:
 
-    results = {'functest': {}, 'yardstick': {}}
+    results = {'functest': {}, 'yardstick': {}, 'shell': {}}
 
     logger = None
 
@@ -40,8 +40,9 @@ class Report:
 
     @staticmethod
     def check_result(testcase, db_result):
-        checker = CheckerFactory.create(testcase.script_type())
-        checker.check(testcase, db_result)
+        checker = CheckerFactory.create(testcase.validate_type())
+        if checker is not None:
+            checker.check(testcase, db_result)
 
     @classmethod
     def generate_json(cls, testsuite_yaml, testarea, duration):
@@ -163,24 +164,26 @@ class Report:
 
     @classmethod
     def get_result(cls, testcase):
-        script_testcase = testcase.script_testcase()
-        type = testcase.script_type()
+        validate_testcase = testcase.validate_testcase()
+        type = testcase.validate_type()
         crawler = CrawlerFactory.create(type)
+        if crawler is None:
+            return None
 
-        if script_testcase in cls.results[type]:
-            return cls.results[type][script_testcase]
+        if validate_testcase in cls.results[type]:
+            return cls.results[type][validate_testcase]
 
-        result = crawler.crawl(script_testcase)
+        result = crawler.crawl(validate_testcase)
 
         if result is not None:
-            cls.results[type][script_testcase] = result
+            cls.results[type][validate_testcase] = result
             testcase.script_result_acquired(True)
             cls.logger.debug('testcase: %s -> result acquired' %
-                             script_testcase)
+                             validate_testcase)
         else:
             retry = testcase.increase_retry()
             cls.logger.debug('testcase: %s -> result acquired retry:%d' %
-                             (script_testcase, retry))
+                             (validate_testcase, retry))
         return result