Use template to unify commands in functest/yardstick

JIRA: DOVETAIL-19

1. use jinja2 to unify commands in config files
2. it simplify the process of test execution, put the dissimilarity in config
3. add precondition/postcondition for functest/yardstick config

Change-Id: Ib996b11ea065b61910b34b78191bb7b1ffd92e59
Signed-off-by: Leo Wang <grakiss.wanglei@huawei.com>

12 files changed, 191 insertions, 33 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 8d6883c1..bdf5a40d 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,23 +2,36 @@ FROM ubuntu:14.04
 MAINTAINER Leo Wang <grakiss.wanglei@huawei.com>
 LABEL version="0.1" description="OPNFV Dovetail Docker Container"
 
-RUN apt-get update && apt-get install -y \
-docker.io \
-python-pip \
-git \
---no-install-recommends
-
-RUN pip install pyyaml
-RUN pip install click
+RUN \
+    apt-get update \
+&& \
+    apt-get install -y \
+        python-pip \
+        git \
+        apt-transport-https \
+        --no-install-recommends \
+&& \
+    apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D \
+&& \
+    mkdir -p /etc/apt/sources.list.d \
+&& \
+    echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list \
+&& \
+    apt-get update &&  apt-get install -y docker-engine=1.9.1-0~trusty \
+&& \
+    pip install pyyaml \
+        click \
+        jinja2
 
 ENV HOME /home/opnfv
 ENV REPOS_DIR /home/opnfv/dovetail
 WORKDIR /home/opnfv
 
-RUN git config --global http.sslVerify false
-RUN git clone https://gerrit.opnfv.org/gerrit/dovetail.git ${REPOS_DIR}
-
-RUN mkdir -p ${REPOS_DIR}/results
+RUN \
+    git config --global http.sslVerify false \
+&& \
+    git clone https://gerrit.opnfv.org/gerrit/dovetail.git ${REPOS_DIR} \
+&& \
+    mkdir -p ${REPOS_DIR}/results
 
 WORKDIR /home/opnfv/dovetail/scripts
-
diff --git a/scripts/conf/dovetail_config.py b/scripts/conf/dovetail_config.py
index 92d1f479..e7942f52 100644
--- a/scripts/conf/dovetail_config.py
+++ b/scripts/conf/dovetail_config.py
@@ -17,7 +17,7 @@ import os
 with open(os.path.join(os.getcwd(),'conf','dovetail_config.yml')) as f:
     dovetail_config = yaml.safe_load(f)
 
-for extra_config_file in dovetail_config['extra_config']:
+for extra_config_file in dovetail_config['include_config']:
     with open(os.path.join(os.getcwd(),'conf',extra_config_file)) as f:
         extra_config = yaml.safe_load(f)
         dovetail_config.update(extra_config)
diff --git a/scripts/conf/dovetail_config.yml b/scripts/conf/dovetail_config.yml
index 57d6e894..901988f8 100644
--- a/scripts/conf/dovetail_config.yml
+++ b/scripts/conf/dovetail_config.yml
@@ -3,7 +3,17 @@ work_dir: /home/opnfv/dovetail
 result_dir: /home/opnfv/dovetail/results
 report_file: 'dovetail_report.txt'
 
-extra_config:
+# used for testcase cmd template in jinja2 format
+# we have two variables available now
+# parameter path, use this path to walk through python object and get value
+# and the python object is "testcase" object by hard-coded
+parameters:
+  - name: testcase
+    path: '("name",)'
+  - name: script_testcase
+    path: '("scripts", "testcase")'
+
+include_config:
   - functest_config.yml
   - yardstick_config.yml
 
diff --git a/scripts/conf/functest_config.yml b/scripts/conf/functest_config.yml
index 86e6ce75..cd33dc59 100644
--- a/scripts/conf/functest_config.yml
+++ b/scripts/conf/functest_config.yml
@@ -6,10 +6,19 @@ functest:
          -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal'
   opts: '-id --privileged=true'
   result_dir: '/home/opnfv/functest/results'
+  pre_condition:
+    cmds:
+      - 'echo test for precondition'
   testcase:
     pre_cmd: 'python /home/opnfv/repos/functest/ci/prepare_env.py start'
-    exec_cmd: 'python /home/opnfv/repos/functest/ci/run_tests.py -t %s -r'
+    exec_cmd: 'python /home/opnfv/repos/functest/ci/run_tests.py -t {{script_testcase}} -r'
     post_cmd: ''
+    cmds:
+      - 'python /home/opnfv/repos/functest/ci/prepare_env.py start'
+      - 'python /home/opnfv/repos/functest/ci/run_tests.py -t {{script_testcase}} -r'
+  post_condition:
+    cmds:
+      - ''
   result:
     dir: '/home/opnfv/functest/results'
     store_type: 'file'
diff --git a/scripts/conf/yardstick_config.yml b/scripts/conf/yardstick_config.yml
index 9eda6e54..7a40dc70 100644
--- a/scripts/conf/yardstick_config.yml
+++ b/scripts/conf/yardstick_config.yml
@@ -9,9 +9,18 @@ yardstick:
   result_dir: '/tmp/yardstick/result'
   shell_dir: '/tmp/yardstick'
   shell_dir_name: 'prepare_test_yard'
+  pre_condition:
+    cmds:
+      - 'echo test for precondition'
   testcase:
     build_test_cmd: '/tmp/yardstick/build_run_test.sh %s.yaml /tmp/yardstick/result/%s.out'
     test_cmd: '/tmp/yardstick/run_test.sh %s.yaml /tmp/yardstick/result/%s.out'
+    cmds:
+      - '/tmp/yardstick/build_run_test.sh {{script_testcase}}.yaml /tmp/yardstick/result/{{testcase}}.out'
+      - '/tmp/yardstick/run_test.sh {{script_testcase}}.yaml /tmp/yardstick/result/{{testcase}}.out'
+  post_condition:
+    cmds:
+      - ''
   result:
     dir: '/tmp/yardstick/result'
     store_type: 'file'
diff --git a/scripts/container.py b/scripts/container.py
index 678a0929..3223c993 100644
--- a/scripts/container.py
+++ b/scripts/container.py
@@ -66,7 +66,7 @@ class Container:
 
     @classmethod
     def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
-        cmd = 'sudo docker exec %s %s' % (container_id, sub_cmd)
+        cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd)
         dt_utils.exec_cmd(cmd,logger,exit_on_error)
 
     @classmethod
diff --git a/scripts/parser.py b/scripts/parser.py
index c0b18e13..1c0c0450 100644
--- a/scripts/parser.py
+++ b/scripts/parser.py
@@ -7,3 +7,34 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
+import jinja2
+
+
+import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
+
+logger = dt_logger.Logger('parser.py').getLogger()
+
+from conf.dovetail_config import *
+
+class Parser:
+    '''preprocess configuration files'''
+
+    @classmethod
+    def parse_cmd(cls, cmd, testcase):
+        cmd_lines = None
+        try:
+            template = jinja2.Template(cmd, undefined=jinja2.StrictUndefined)
+            kwargs = {}
+            for arg in dovetail_config['parameters']:
+                path = eval(arg['path'])
+                logger.debug('name: %s, eval path: %s ' % (arg['name'], path))
+                kwargs[arg['name']] = dt_utils.get_obj_by_path(testcase.testcase,path)
+
+            logger.debug('kwargs: %s' %  kwargs)
+            cmd_lines = template.render(**kwargs)
+        except Exception as e:
+            logger.error('failed to parse cmd %s, exception:%s' % (cmd, e))
+            return None
+
+        return cmd_lines
diff --git a/scripts/prepare_env.py b/scripts/prepare_env.py
index f3915a95..bd484302 100644
--- a/scripts/prepare_env.py
+++ b/scripts/prepare_env.py
@@ -18,6 +18,6 @@ logger = dt_logger.Logger('prepare_env.py').getLogger()
 cmd = "sudo apt-get -y install docker.io python-pip"
 dt_utils.exec_cmd(cmd, logger)
 
-cmd = "sudo pip install click pyyaml"
+cmd = "sudo pip install click pyyaml jinja2"
 dt_utils.exec_cmd(cmd, logger)
 
diff --git a/scripts/report.py b/scripts/report.py
index d92929f2..42ec0df0 100644
--- a/scripts/report.py
+++ b/scripts/report.py
@@ -13,11 +13,11 @@ import re
 import utils.dovetail_logger as dt_logger
 import utils.dovetail_utils as dt_utils
 
-logger = dt_logger.Logger('report.py').getLogger()
-
 from conf.dovetail_config import *
 from testcase import *
 
+logger = dt_logger.Logger('report.py').getLogger()
+
 def get_pass_str(passed):
     if passed:
         return 'PASS'
diff --git a/scripts/run.py b/scripts/run.py
index 9b59f76f..2afdfc7e 100755
--- a/scripts/run.py
+++ b/scripts/run.py
@@ -15,13 +15,15 @@ import time
 
 import utils.dovetail_logger as dt_logger
 import utils.dovetail_utils as dt_utils
-logger = dt_logger.Logger('run.py').getLogger()
+
 
 from container import Container
 from testcase import *
 from report import *
 from conf.dovetail_config import *
 
+logger = dt_logger.Logger('run.py').getLogger()
+
 def load_scenario(scenario):
     Scenario.load()
     return Scenario.get(SCENARIO_NAMING_FMT % scenario)
@@ -30,10 +32,8 @@ def load_testcase():
     Testcase.load()
 
 def run_functest(testcase, container_id):
-    sub_cmd = dovetail_config[testcase.script_type()]['testcase']['pre_cmd']
-    Container.exec_cmd(container_id, sub_cmd)
-    sub_cmd = dovetail_config[testcase.script_type()]['testcase']['exec_cmd'] % testcase.script_testcase()
-    Container.exec_cmd(container_id, sub_cmd)
+    for cmd in testcase.cmds:
+        Container.exec_cmd(container_id, cmd)
 
 def run_yardstick(testcase, container_id):
     type = testcase.script_type()
@@ -51,23 +51,35 @@ def run_test(scenario):
     for testcase_name in scenario['testcase_list']:
         logger.info('>>[testcase]: %s' % (testcase_name))
         testcase = Testcase.get(testcase_name)
-        run_test = True
+        run_testcase = True
 
         if testcase.exceed_max_retry_times():
-            run_test = False
+            run_testcase = False
 
         if testcase.script_result_acquired():
-            run_test = False
+            run_testcase = False
 
-        if run_test:
+        if run_testcase:
             Container.pull_image(testcase.script_type())
             container_id = Container.create(testcase.script_type())
             logger.debug('container id:%s' % container_id)
 
-            if testcase.script_type() == 'functest':
-                run_functest(testcase, container_id)
+            if not Testcase.prepared(testcase.script_type()):
+                cmds = Testcase.pre_condition(testcase.script_type())['cmds']
+                if cmds:
+                    for cmd in cmds:
+                        Container.exec_cmd(container_id, cmd)
+                Testcase.prepared(testcase.script_type(),True)
+
+            if not testcase.prepare_cmd():
+                logger.error('failed to prepare testcase:%s' % testcase.name())
             else:
-                run_yardstick(testcase, container_id)
+                if testcase.script_type() == 'functest':
+                    run_functest(testcase, container_id)
+                else:
+                    run_yardstick(testcase, container_id)
+
+            #testcase.post_condition()
 
             Container.clean(container_id)
 
diff --git a/scripts/testcase.py b/scripts/testcase.py
index 71ffd7ac..4deabe2e 100644
--- a/scripts/testcase.py
+++ b/scripts/testcase.py
@@ -7,9 +7,13 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
+import jinja2
+
 import utils.dovetail_logger as dt_logger
 import utils.dovetail_utils as dt_utils
 
+from  parser import *
+
 logger = dt_logger.Logger('testcase.py').getLogger()
 
 from conf.dovetail_config import *
@@ -19,9 +23,19 @@ class Testcase:
     def __init__(self, testcase_yaml):
         self.testcase = testcase_yaml.values()[0]
         self.testcase['passed'] = False
+        self.cmds = []
         self.sub_testcase_status = {}
         Testcase.update_script_testcase(self.script_type(), self.script_testcase())
 
+    def prepare_cmd(self):
+        for cmd in dovetail_config[self.script_type()]['testcase']['cmds']:
+            cmd_lines = Parser.parse_cmd(cmd,self)
+            if not cmd_lines:
+                return False
+            self.cmds.append(cmd_lines)
+
+        return True
+
     def __str__(self):
         return self.testcase
 
@@ -63,6 +77,13 @@ class Testcase:
     def script_result_acquired(self, acquired=None):
         return Testcase._result_acquired(self.script_type(), self.script_testcase(), acquired)
 
+    def pre_condition(self):
+        return Testcase.pre_condition(self.script_type())
+
+    def post_condition(self):
+        return Testcase.post_condition(self.script_type())
+
+
     #testcase in upstream testing project
     script_testcase_list = {'functest':{}, 'yardstick':{}}
 
@@ -70,9 +91,31 @@ class Testcase:
     testcase_list = {}
 
     @classmethod
+    def prepared(cls, script_type, prepared=None):
+        if prepared is not None:
+            cls.script_testcase_list[script_type]['prepared'] = prepared
+        return cls.script_testcase_list[script_type]['prepared']
+
+    @classmethod
+    def cleaned(cls, script_type, cleaned=None):
+        if cleaned is not None:
+            cls.scrpit_testcase_list[script_type]['cleaned'] = cleaned
+        return cls.script_testcase_list[script_type]['cleaned']
+
+    @classmethod
+    def pre_condition(cls, script_type):
+        return dovetail_config[script_type]['pre_condition']
+
+    def post_condition(cls, script_type):
+        return dovetail_config[script_type]['post_condition']
+
+
+    @classmethod
     def update_script_testcase(cls,script_type, script_testcase):
         if script_testcase not in cls.script_testcase_list[script_type]:
             cls.script_testcase_list[script_type][script_testcase] = {'retry':0, 'acquired':False}
+            cls.script_testcase_list[script_type]['prepared'] = False
+            cls.script_testcase_list[script_type]['cleaned'] = False
 
     @classmethod
     def _exceed_max_retry_times(cls, script_type, script_testcase ):
diff --git a/scripts/utils/dovetail_utils.py b/scripts/utils/dovetail_utils.py
index f79c6fc4..4c671552 100644
--- a/scripts/utils/dovetail_utils.py
+++ b/scripts/utils/dovetail_utils.py
@@ -32,7 +32,7 @@ def exec_cmd(cmd, logger=None,
                          stderr=subprocess.STDOUT)
     output = p.communicate()
     for line in output[0].strip().split('\n'):
-        line = line.replace('\n', '') 
+        line = line.replace('\n', '')
         if logger:
             if info:
                 logger.info(line)
@@ -54,3 +54,34 @@ def exec_cmd(cmd, logger=None,
 
     return returncode, output[0].strip()
 
+
+#walkthrough the object, yield path and value
+from collections import Mapping, Set, Sequence
+
+# dual python 2/3 compatability, inspired by the "six" library
+string_types = (str, unicode) if str is bytes else (str, bytes)
+iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()
+
+def objwalk(obj, path=(), memo=None):
+    if memo is None:
+        memo = set()
+    iterator = None
+    if isinstance(obj, Mapping):
+        iterator = iteritems
+    elif isinstance(obj, (Sequence, Set)) and not isinstance(obj, string_types):
+        iterator = enumerate
+    if iterator:
+        if id(obj) not in memo:
+            memo.add(id(obj))
+            for path_component, value in iterator(obj):
+                for result in objwalk(value, path + (path_component,), memo):
+                    yield result
+            memo.remove(id(obj))
+    else:
+        yield path, obj
+
+def get_obj_by_path(obj,dst_path):
+    for path, obj in objwalk(obj):
+        if path == dst_path:
+            return obj
+