summaryrefslogtreecommitdiffstats
path: root/dovetail
diff options
context:
space:
mode:
Diffstat (limited to 'dovetail')
-rw-r--r--dovetail/compliance/compliance_set.yml (renamed from dovetail/cert/compliance_set.yml)4
-rw-r--r--dovetail/compliance/debug.yml9
-rw-r--r--dovetail/compliance/proposed_tests.yml (renamed from dovetail/cert/proposed_tests.yml)4
-rw-r--r--dovetail/conf/cmd_config.yml18
-rw-r--r--dovetail/conf/dovetail_config.py16
-rw-r--r--dovetail/conf/dovetail_config.yml2
-rw-r--r--dovetail/container.py30
-rw-r--r--dovetail/parser.py19
-rw-r--r--dovetail/report.py221
-rwxr-xr-xdovetail/run.py93
-rw-r--r--dovetail/testcase.py44
-rw-r--r--dovetail/tests/unit/test_parser.py3
-rw-r--r--dovetail/utils/dovetail_logger.py14
13 files changed, 322 insertions, 155 deletions
diff --git a/dovetail/cert/compliance_set.yml b/dovetail/compliance/compliance_set.yml
index b4108b4f..df0d427d 100644
--- a/dovetail/cert/compliance_set.yml
+++ b/dovetail/compliance/compliance_set.yml
@@ -1,5 +1,5 @@
-certification_compliance_set:
- name: certification_compliance_set
+compliance_set:
+ name: compliance_set
testcases_list:
# Temporarily, one test case kept here as default to run
# for use of software development/debug
diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml
new file mode 100644
index 00000000..a0a2d3ee
--- /dev/null
+++ b/dovetail/compliance/debug.yml
@@ -0,0 +1,9 @@
+# only used for dovetail tool development debug
+# nfvi.tc001 tc002 running time is shorter, about 3 minutes
+# ipv6.tc001 about 20 minutes
+debug:
+ name: debug
+ testcases_list:
+ - dovetail.ipv6.tc001
+ - dovetail.nfvi.tc001
+ - dovetail.nfvi.tc002
diff --git a/dovetail/cert/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 6d6c8d19..fcb7e03a 100644
--- a/dovetail/cert/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -1,5 +1,5 @@
-certification_proposed_tests:
- name: certification_proposed_tests
+proposed_tests:
+ name: proposed_tests
testcases_list:
# TO DO: will adjust the dovetail tool to support in later patches
# run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc}
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index c2108c58..4e3d0110 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -31,18 +31,22 @@ cli:
- '--DEPLOY_TYPE'
- '-T'
help: 'DEPLOY_TYPE of the system under test (SUT).'
- CI_DEBUG:
+ DEBUG:
flags:
- - '--CI_DEBUG'
+ - '--DEBUG'
- '-d'
- help: 'CI_DEBUG for showing debug log.'
+ help: 'DEBUG for showing debug log.'
non-envs:
- scenario:
+ testsuite:
flags:
- - '--scenario'
- - '-s'
+ - '--testsuite'
default: 'compliance_set'
- help: 'certification scenario.'
+ help: 'compliance testsuite.'
+ testarea:
+ flags:
+ - '--testarea'
+ default: 'full'
+ help: 'compliance testarea within testsuite'
tag:
flags:
- '--tag'
diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py
index 5ac23c43..e5d0608e 100644
--- a/dovetail/conf/dovetail_config.py
+++ b/dovetail/conf/dovetail_config.py
@@ -14,9 +14,12 @@ import re
class DovetailConfig:
- CERT_PATH = './cert/'
+ COMPLIANCE_PATH = './compliance/'
TESTCASE_PATH = './testcase/'
- SCENARIO_NAMING_FMT = 'certification_%s'
+ # testsuite supported tuple, should adjust accordingly
+ testsuite_supported = ('compliance_set', 'proposed_tests', 'debug')
+ # testarea supported tuple, should adjust accordingly
+ testarea_supported = ('vimops', 'nfvi', 'ipv6')
curr_path = os.path.dirname(os.path.abspath(__file__))
@@ -44,14 +47,17 @@ class DovetailConfig:
@classmethod
def update_envs(cls, options):
for item in options:
- if options[item] is not None:
- key = cls.cmd_name_trans(item)
- os.environ[key] = options[item]
+ key = cls.cmd_name_trans(item)
+ if not options[item] and key in os.environ:
+ options[item] = os.environ[key]
+ if options[item]:
cls.update_config_envs('functest', key)
cls.update_config_envs('yardstick', key)
@classmethod
def update_config_envs(cls, script_type, key):
+ if key == 'DEBUG':
+ os.environ['CI_DEBUG'] = os.environ[key]
envs = cls.dovetail_config[script_type]['envs']
old_value = re.findall(r'\s+%s=(.*?)(\s+|$)' % key, envs)
if old_value == []:
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index 1f5de672..c4131a91 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -3,6 +3,8 @@ work_dir: /home/opnfv/dovetail
result_dir: /home/opnfv/dovetail/results
report_file: 'dovetail_report.txt'
cli_file_name: 'cmd_config.yml'
+# TO DO: once version scheme settled, adjust accordingly
+repo: 'https://github.com/opnfv/dovetail/tree/master/'
# used for testcase cmd template in jinja2 format
# we have two variables available now
diff --git a/dovetail/container.py b/dovetail/container.py
index 15ccc800..2716a089 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -11,7 +11,6 @@
import utils.dovetail_logger as dt_logger
import utils.dovetail_utils as dt_utils
from conf.dovetail_config import DovetailConfig as dt_config
-logger = dt_logger.Logger('container.py').getLogger()
class Container:
@@ -19,6 +18,8 @@ class Container:
container_list = {}
has_pull_latest_image = {'yardstick': False, 'functest': False}
+ logger = None
+
def __init__(cls):
pass
@@ -26,6 +27,10 @@ class Container:
pass
@classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.Container').getLogger()
+
+ @classmethod
def get(cls, type):
return cls.container_list[type]
@@ -46,10 +51,10 @@ class Container:
dovetail_config[type]['result']['dir'])
cmd = 'sudo docker run %s %s %s %s %s /bin/bash' % \
(opts, envs, sshkey, result_volume, docker_image)
- dt_utils.exec_cmd(cmd, logger)
+ dt_utils.exec_cmd(cmd, cls.logger)
ret, container_id = \
dt_utils.exec_cmd("sudo docker ps | grep " + docker_image +
- " | awk '{print $1}' | head -1", logger)
+ " | awk '{print $1}' | head -1", cls.logger)
cls.container_list[type] = container_id
return container_id
@@ -57,20 +62,21 @@ class Container:
def pull_image(cls, type):
docker_image = cls.get_docker_image(type)
if cls.has_pull_latest_image[type] is True:
- logger.debug('%s is already the newest version.' % (docker_image))
+ cls.logger.debug('%s is already the newest version.' %
+ (docker_image))
else:
cmd = 'sudo docker pull %s' % (docker_image)
- dt_utils.exec_cmd(cmd, logger)
+ dt_utils.exec_cmd(cmd, cls.logger)
cls.has_pull_latest_image[type] = True
- @staticmethod
- def clean(container_id):
+ @classmethod
+ def clean(cls, container_id):
cmd1 = 'sudo docker stop %s' % (container_id)
- dt_utils.exec_cmd(cmd1, logger)
+ dt_utils.exec_cmd(cmd1, cls.logger)
cmd2 = 'sudo docker rm %s' % (container_id)
- dt_utils.exec_cmd(cmd2, logger)
+ dt_utils.exec_cmd(cmd2, cls.logger)
- @staticmethod
- def exec_cmd(container_id, sub_cmd, exit_on_error=False):
+ @classmethod
+ def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd)
- dt_utils.exec_cmd(cmd, logger, exit_on_error)
+ dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
diff --git a/dovetail/parser.py b/dovetail/parser.py
index 1740944a..d8f9fa0a 100644
--- a/dovetail/parser.py
+++ b/dovetail/parser.py
@@ -14,28 +14,33 @@ import utils.dovetail_logger as dt_logger
import utils.dovetail_utils as dt_utils
from conf.dovetail_config import DovetailConfig as dt_config
-logger = dt_logger.Logger('parser.py').getLogger()
-
class Parser:
'''preprocess configuration files'''
- @staticmethod
- def parse_cmd(cmd, testcase):
+ logger = None
+
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.Parser').getLogger()
+
+ @classmethod
+ def parse_cmd(cls, cmd, testcase):
cmd_lines = None
try:
template = jinja2.Template(cmd, undefined=jinja2.StrictUndefined)
kwargs = {}
for arg in dt_config.dovetail_config['parameters']:
path = eval(arg['path'])
- logger.debug('name: %s, eval path: %s ' % (arg['name'], path))
+ cls.logger.debug('name: %s, eval path: %s ' %
+ (arg['name'], path))
kwargs[arg['name']] = \
dt_utils.get_obj_by_path(testcase.testcase, path)
- logger.debug('kwargs: %s' % kwargs)
+ cls.logger.debug('kwargs: %s' % kwargs)
cmd_lines = template.render(**kwargs)
except Exception as e:
- logger.error('failed to parse cmd %s, exception:%s' % (cmd, e))
+ cls.logger.error('failed to parse cmd %s, exception:%s' % (cmd, e))
return None
return cmd_lines
diff --git a/dovetail/report.py b/dovetail/report.py
index 7dd8ba2d..7fd4076d 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -6,18 +6,20 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
+from __future__ import division
+
import json
import urllib2
import re
import os
+import datetime
+import uuid
import utils.dovetail_logger as dt_logger
-from conf.dovetail_config import DovetailConfig as dt_config
+from conf.dovetail_config import DovetailConfig as dt_cfg
from testcase import Testcase
-logger = dt_logger.Logger('report.py').getLogger()
-
def get_pass_str(passed):
if passed:
@@ -30,78 +32,133 @@ class Report:
results = {'functest': {}, 'yardstick': {}}
+ logger = None
+
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.Report').getLogger()
+
@staticmethod
def check_result(testcase, db_result):
checker = CheckerFactory.create(testcase.script_type())
checker.check(testcase, db_result)
@classmethod
- def generate_json(cls, scenario_yaml):
+ def generate_json(cls, testsuite_yaml, testarea, duration):
report_obj = {}
- report_obj['scenario'] = scenario_yaml['name']
+ # TO DO: once version scheme settled, adjust accordingly
+ report_obj['version'] = '1.0'
+ report_obj['testsuite'] = testsuite_yaml['name']
+ # TO DO: once dashboard url settled, adjust accordingly
+ report_obj['dashboard'] = None
+ report_obj['validation_ID'] = str(uuid.uuid4())
+ report_obj['upload_date'] =\
+ datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
+ report_obj['duration'] = duration
+
report_obj['testcases_list'] = []
- for testcase_name in scenario_yaml['testcases_list']:
+ testarea_list = []
+ for value in testsuite_yaml['testcases_list']:
+ if value is not None and (testarea == 'full' or testarea in value):
+ testarea_list.append(value)
+ for testcase_name in testarea_list:
testcase = Testcase.get(testcase_name)
- testcase_in_rpt = {}
- testcase_in_rpt['name'] = testcase_name
+ testcase_inreport = {}
+ testcase_inreport['name'] = testcase_name
if testcase is None:
- testcase_in_rpt['result'] = 'Undefined'
- testcase_in_rpt['objective'] = ''
- testcase_in_rpt['sub_testcase'] = []
- report_obj['testcases_list'].append(testcase_in_rpt)
+ testcase_inreport['result'] = 'Undefined'
+ testcase_inreport['objective'] = ''
+ testcase_inreport['sub_testcase'] = []
+ report_obj['testcases_list'].append(testcase_inreport)
continue
- testcase_in_rpt['result'] = get_pass_str(testcase.passed())
- testcase_in_rpt['objective'] = testcase.objective()
- testcase_in_rpt['sub_testcase'] = []
+ testcase_inreport['result'] = get_pass_str(testcase.passed())
+ testcase_inreport['objective'] = testcase.objective()
+ testcase_inreport['sub_testcase'] = []
if testcase.sub_testcase() is not None:
for sub_test in testcase.sub_testcase():
- testcase_in_rpt['sub_testcase'].append({
+ testcase_inreport['sub_testcase'].append({
'name': sub_test,
'result': get_pass_str(
testcase.sub_testcase_passed(sub_test))
})
- report_obj['testcases_list'].append(testcase_in_rpt)
- logger.info(json.dumps(report_obj))
+ report_obj['testcases_list'].append(testcase_inreport)
+ cls.logger.info(json.dumps(report_obj))
return report_obj
@classmethod
- def generate(cls, scenario_yaml):
- rpt_data = cls.generate_json(scenario_yaml)
- rpt_text = ''
- split_line = '+-----------------------------------------------------'
- split_line += '---------------------+\n'
-
- rpt_text += '\n\
-+==========================================================================+\n\
-| report |\n'
- rpt_text += split_line
- rpt_text += '|scenario: %s\n' % rpt_data['scenario']
- for testcase in rpt_data['testcases_list']:
- rpt_text += '| [testcase]: %s\t\t\t\t[%s]\n' % \
- (testcase['name'], testcase['result'])
- rpt_text += '| |-objective: %s\n' % testcase['objective']
- if 'sub_testcase' in testcase:
- for sub_test in testcase['sub_testcase']:
- rpt_text += '| |-%s \t\t [%s]\n' % \
- (sub_test['name'], sub_test['result'])
- rpt_text += split_line
-
- logger.info(rpt_text)
- cls.save(rpt_text)
- return rpt_text
+ def generate(cls, testsuite_yaml, testarea, duration):
+ report_data = cls.generate_json(testsuite_yaml, testarea, duration)
+ report_txt = ''
+ report_txt += '\n\nDovetail Report\n'
+ report_txt += 'Version: %s\n' % report_data['version']
+ report_txt += 'TestSuite: %s\n' % report_data['testsuite']
+ report_txt += 'Result Dashboard: %s\n' % report_data['dashboard']
+ report_txt += 'Validation ID: %s\n' % report_data['validation_ID']
+ report_txt += 'Upload Date: %s\n' % report_data['upload_date']
+ if report_data['duration'] == 0:
+ report_txt += 'Duration: %s\n\n' % 'NA'
+ else:
+ report_txt += 'Duration: %.2f s\n\n' % report_data['duration']
+
+ total_num = 0
+ pass_num = 0
+ sub_report = {}
+ testcase_num = {}
+ testcase_passnum = {}
+ for area in dt_cfg.testarea_supported:
+ sub_report[area] = ''
+ testcase_num[area] = 0
+ testcase_passnum[area] = 0
+
+ # TO DO: once version scheme settled, adjust accordingly
+ spec_link = dt_cfg.dovetail_config['repo'] + 'dovetail/testcase'
+ for testcase in report_data['testcases_list']:
+ pattern = re.compile('|'.join(dt_cfg.testarea_supported))
+ area = pattern.findall(testcase['name'])[0]
+ result_dir = dt_cfg.dovetail_config['result_dir']
+ sub_report[area] += '- <%s> %s result: <%s>\n' %\
+ (spec_link, testcase['name'], result_dir)
+ testcase_num[area] += 1
+ total_num += 1
+ if testcase['result'] == 'PASS':
+ testcase_passnum[area] += 1
+ pass_num += 1
+
+ if total_num != 0:
+ pass_rate = pass_num/total_num
+ report_txt += 'Pass Rate: %.2f%% (%s/%s)\n' %\
+ (pass_rate*100, pass_num, total_num)
+ report_txt += 'Assessed test areas:\n'
+ for key in sub_report:
+ if testcase_num[key] != 0:
+ pass_rate = testcase_passnum[key]/testcase_num[key]
+ # TO DO: once version scheme settled, adjust accordingly
+ doc_link = dt_cfg.dovetail_config['repo'] +\
+ ('docs/testsuites/%s' % key)
+ report_txt += '- %s results: <%s> pass %.2f%%\n' %\
+ (key, doc_link, pass_rate*100)
+ for key in sub_report:
+ if testcase_num[key] != 0:
+ pass_rate = testcase_passnum[key]/testcase_num[key]
+ report_txt += '%s: pass rate %.2f%%\n' % (key, pass_rate*100)
+ report_txt += sub_report[key]
+
+ cls.logger.info(report_txt)
+ cls.save(report_txt)
+ return report_txt
# save to disk as default
- @staticmethod
- def save(report):
- report_file_name = dt_config.dovetail_config['report_file']
+ @classmethod
+ def save(cls, report):
+ report_file_name = dt_cfg.dovetail_config['report_file']
try:
- with open(os.path.join(dt_config.dovetail_config['result_dir'],
+ with open(os.path.join(dt_cfg.dovetail_config['result_dir'],
report_file_name), 'w') as report_file:
report_file.write(report)
- logger.info('save report to %s' % report_file_name)
+ cls.logger.info('save report to %s' % report_file_name)
except Exception:
- logger.error('Failed to save: %s' % report_file_name)
+ cls.logger.error('Failed to save: %s' % report_file_name)
@classmethod
def get_result(cls, testcase):
@@ -117,11 +174,12 @@ class Report:
if result is not None:
cls.results[type][script_testcase] = result
testcase.script_result_acquired(True)
- logger.debug('testcase: %s -> result acquired' % script_testcase)
+ cls.logger.debug('testcase: %s -> result acquired' %
+ script_testcase)
else:
retry = testcase.increase_retry()
- logger.debug('testcase: %s -> result acquired retry:%d' %
- (script_testcase, retry))
+ cls.logger.debug('testcase: %s -> result acquired retry:%d' %
+ (script_testcase, retry))
return result
@@ -140,12 +198,18 @@ class CrawlerFactory:
class FunctestCrawler:
+ logger = None
+
def __init__(self):
self.type = 'functest'
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.FunctestCrawler').getLogger()
+
def crawl(self, testcase=None):
store_type = \
- dt_config.dovetail_config[self.type]['result']['store_type']
+ dt_cfg.dovetail_config[self.type]['result']['store_type']
if store_type == 'file':
return self.crawl_from_file(testcase)
@@ -153,12 +217,12 @@ class FunctestCrawler:
return self.crawl_from_url(testcase)
def crawl_from_file(self, testcase=None):
- dovetail_config = dt_config.dovetail_config
+ dovetail_config = dt_cfg.dovetail_config
file_path = \
os.path.join(dovetail_config['result_dir'],
dovetail_config[self.type]['result']['file_path'])
if not os.path.exists(file_path):
- logger.info('result file not found: %s' % file_path)
+ self.logger.info('result file not found: %s' % file_path)
return None
try:
@@ -180,34 +244,40 @@ class FunctestCrawler:
"duration": int(dur_sec_int),
"tests": int(num_tests), "failures": failed_num,
"errors": error_logs}}
- logger.debug('Results: %s' % str(json_results))
+ self.logger.debug('Results: %s' % str(json_results))
return json_results
except Exception as e:
- logger.error('Cannot read content from the file: %s, exception: %s'
- % (file_path, e))
+ self.logger.error('Cannot read content from the file: %s, '
+ 'exception: %s' % (file_path, e))
return None
def crawl_from_url(self, testcase=None):
url = \
- dt_config.dovetail_config[self.type]['result']['db_url'] % testcase
- logger.debug("Query to rest api: %s" % url)
+ dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase
+ self.logger.debug("Query to rest api: %s" % url)
try:
data = json.load(urllib2.urlopen(url))
return data['results'][0]
except Exception as e:
- logger.error("Cannot read content from the url: %s, exception: %s"
- % (url, e))
+ self.logger.error("Cannot read content from the url: %s, "
+ "exception: %s" % (url, e))
return None
class YardstickCrawler:
+ logger = None
+
def __init__(self):
self.type = 'yardstick'
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.YardstickCrawler').getLogger()
+
def crawl(self, testcase=None):
store_type = \
- dt_config.dovetail_config[self.type]['result']['store_type']
+ dt_cfg.dovetail_config[self.type]['result']['store_type']
if store_type == 'file':
return self.crawl_from_file(testcase)
@@ -215,21 +285,21 @@ class YardstickCrawler:
return self.crawl_from_url(testcase)
def crawl_from_file(self, testcase=None):
- file_path = os.path.join(dt_config.dovetail_config['result_dir'],
+ file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
testcase+'.out')
if not os.path.exists(file_path):
- logger.info('result file not found: %s' % file_path)
+ self.logger.info('result file not found: %s' % file_path)
return None
try:
with open(file_path, 'r') as myfile:
myfile.read()
criteria = 'PASS'
json_results = {'criteria': criteria}
- logger.debug('Results: %s' % str(json_results))
+ self.logger.debug('Results: %s' % str(json_results))
return json_results
except Exception as e:
- logger.error('Cannot read content from the file: %s, exception: %s'
- % (file_path, e))
+ self.logger.error('Cannot read content from the file: %s, '
+ 'exception: %s' % (file_path, e))
return None
def crawl_from_url(self, testcase=None):
@@ -258,8 +328,13 @@ class ResultChecker:
class FunctestChecker:
- @staticmethod
- def check(testcase, db_result):
+ logger = None
+
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.FunctestChecker').getLogger()
+
+ def check(self, testcase, db_result):
sub_testcase_list = testcase.sub_testcase()
if not db_result:
@@ -280,7 +355,7 @@ class FunctestChecker:
all_passed = True
for sub_testcase in sub_testcase_list:
- logger.debug('check sub_testcase:%s' % sub_testcase)
+ self.logger.debug('check sub_testcase:%s' % sub_testcase)
if sub_testcase in db_result['details']['errors']:
testcase.sub_testcase_passed(sub_testcase, False)
all_passed = False
@@ -292,6 +367,12 @@ class FunctestChecker:
class YardstickChecker:
+ logger = None
+
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.YardstickChecker').getLogger()
+
@staticmethod
def check(testcase, result):
if not result:
diff --git a/dovetail/run.py b/dovetail/run.py
index 86194389..d24919b6 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -10,22 +10,25 @@
import click
import sys
+import os
+import time
import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
-
+from parser import Parser
from container import Container
from testcase import Testcase
-from testcase import Scenario
+from testcase import Testsuite
from report import Report
+from report import FunctestCrawler, YardstickCrawler
+from report import FunctestChecker, YardstickChecker
from conf.dovetail_config import DovetailConfig as dt_config
-logger = dt_logger.Logger('run.py').getLogger()
-
-def load_scenario(scenario):
- Scenario.load()
- return Scenario.get(dt_config.SCENARIO_NAMING_FMT % scenario)
+def load_testsuite(testsuite):
+ Testsuite.load()
+ return Testsuite.get(testsuite)
def set_container_tags(option_str):
@@ -40,12 +43,18 @@ def load_testcase():
Testcase.load()
-def run_test(scenario):
- for testcase_name in scenario['testcases_list']:
+def run_test(testsuite, testarea, logger):
+ testarea_list = []
+ for value in testsuite['testcases_list']:
+ if value is not None and (testarea == 'full' or testarea in value):
+ testarea_list.append(value)
+
+ duration = 0
+ for testcase_name in testarea_list:
logger.info('>>[testcase]: %s' % (testcase_name))
testcase = Testcase.get(testcase_name)
if testcase is None:
- logger.error('testcase %s is not defined in testcase folder, \
+ logger.error('test case %s is not defined in testcase folder, \
skipping' % (testcase_name))
continue
run_testcase = True
@@ -71,8 +80,11 @@ def run_test(scenario):
if not testcase.prepare_cmd():
logger.error('failed to prepare testcase:%s' % testcase.name())
else:
+ start_time = time.time()
for cmd in testcase.cmds:
Container.exec_cmd(container_id, cmd)
+ end_time = time.time()
+ duration = end_time - start_time
# testcase.post_condition()
@@ -81,8 +93,10 @@ def run_test(scenario):
db_result = Report.get_result(testcase)
Report.check_result(testcase, db_result)
+ return duration
-def validate_options(input_dict):
+
+def validate_options(input_dict, logger):
# for 'tag' option
for key, value in input_dict.items():
if key == 'tag' and value is not None:
@@ -101,24 +115,63 @@ def filter_env_options(input_dict):
return envs_options
+def create_logs():
+ Container.create_log()
+ Parser.create_log()
+ Report.create_log()
+ FunctestCrawler.create_log()
+ YardstickCrawler.create_log()
+ FunctestChecker.create_log()
+ YardstickChecker.create_log()
+ Testcase.create_log()
+ Testsuite.create_log()
+
+
+def clean_results_dir():
+ result_path = dt_config.dovetail_config['result_dir']
+ if os.path.exists(result_path):
+ if os.path.isdir(result_path):
+ cmd = 'sudo rm -rf %s/*' % (result_path)
+ dt_utils.exec_cmd(cmd, exit_on_error=False)
+ else:
+ print "result_dir in dovetail_config.yml is not a directory."
+ sys.exit(-1)
+
+
def main(*args, **kwargs):
- """Dovetail certification test entry!"""
- logger.info('=======================================')
- logger.info('Dovetail certification: %s!' % (kwargs['scenario']))
- logger.info('=======================================')
- validate_options(kwargs)
+ """Dovetail compliance test entry!"""
+ clean_results_dir()
+ create_logs()
+ logger = dt_logger.Logger('run').getLogger()
+ logger.info('================================================')
+ logger.info('Dovetail compliance: %s!' % (kwargs['testsuite']))
+ logger.info('================================================')
+ validate_options(kwargs, logger)
envs_options = filter_env_options(kwargs)
dt_config.update_envs(envs_options)
logger.info('Your new envs for functest: %s' %
dt_config.dovetail_config['functest']['envs'])
logger.info('Your new envs for yardstick: %s' %
dt_config.dovetail_config['yardstick']['envs'])
- load_testcase()
- scenario_yaml = load_scenario(kwargs['scenario'])
+
if 'tag' in kwargs and kwargs['tag'] is not None:
set_container_tags(kwargs['tag'])
- run_test(scenario_yaml)
- Report.generate(scenario_yaml)
+
+ testarea = kwargs['testarea']
+ testsuite_validation = False
+ testarea_validation = False
+ if (testarea == 'full') or (testarea in dt_config.testarea_supported):
+ testarea_validation = True
+ if kwargs['testsuite'] in dt_config.testsuite_supported:
+ testsuite_validation = True
+ if testsuite_validation and testarea_validation:
+ testsuite_yaml = load_testsuite(kwargs['testsuite'])
+ load_testcase()
+ duration = run_test(testsuite_yaml, testarea, logger)
+ Report.generate(testsuite_yaml, testarea, duration)
+ else:
+ logger.error('invalid input commands, testsuite %s testarea %s' %
+ (kwargs['testsuite'], testarea))
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 6f0552b6..60ce2b28 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -15,11 +15,11 @@ import utils.dovetail_logger as dt_logger
from parser import Parser
from conf.dovetail_config import DovetailConfig as dt_config
-logger = dt_logger.Logger('testcase.py').getLogger()
-
class Testcase:
+ logger = None
+
def __init__(self, testcase_yaml):
self.testcase = testcase_yaml.values()[0]
self.testcase['passed'] = False
@@ -28,6 +28,10 @@ class Testcase:
self.update_script_testcase(self.script_type(),
self.script_testcase())
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.Testcase').getLogger()
+
def prepare_cmd(self):
script_type = self.script_type()
for cmd in dt_config.dovetail_config[script_type]['testcase']['cmds']:
@@ -52,7 +56,7 @@ class Testcase:
def sub_testcase_passed(self, name, passed=None):
if passed is not None:
- logger.debug('sub_testcase_passed:%s %s' % (name, passed))
+ self.logger.debug('sub_testcase_passed:%s %s' % (name, passed))
self.sub_testcase_status[name] = passed
return self.sub_testcase_status[name]
@@ -146,7 +150,7 @@ class Testcase:
testcase_yaml = yaml.safe_load(f)
cls.testcase_list[testcase_yaml.keys()[0]] = \
cls(testcase_yaml)
- logger.debug(cls.testcase_list)
+ cls.logger.debug(cls.testcase_list)
@classmethod
def get(cls, testcase_name):
@@ -155,31 +159,37 @@ class Testcase:
return None
-class Scenario:
+class Testsuite:
- def __init__(self, scenario):
- self.scenario = scenario
+ logger = None
+
+ def __init__(self, testsuite):
+ self.testsuite = testsuite
self.testcase_list = {}
+ @classmethod
+ def create_log(cls):
+ cls.logger = dt_logger.Logger(__name__+'.Testsuite').getLogger()
+
def get_test(self, testcase_name):
if testcase_name in self.testcase_list:
return self.testcase_list[testcase_name]
return None
- scenario_list = {}
+ testsuite_list = {}
@classmethod
def load(cls):
- for root, dirs, files in os.walk(dt_config.CERT_PATH):
- for scenario_yaml in files:
- with open(os.path.join(root, scenario_yaml)) as f:
- scenario_yaml = yaml.safe_load(f)
- cls.scenario_list.update(scenario_yaml)
+ for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH):
+ for testsuite_yaml in files:
+ with open(os.path.join(root, testsuite_yaml)) as f:
+ testsuite_yaml = yaml.safe_load(f)
+ cls.testsuite_list.update(testsuite_yaml)
- logger.debug(cls.scenario_list)
+ cls.logger.debug(cls.testsuite_list)
@classmethod
- def get(cls, scenario_name):
- if scenario_name in cls.scenario_list:
- return cls.scenario_list[scenario_name]
+ def get(cls, testsuite_name):
+ if testsuite_name in cls.testsuite_list:
+ return cls.testsuite_list[testsuite_name]
return None
diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py
index 5b003d1a..b4331ea1 100644
--- a/dovetail/tests/unit/test_parser.py
+++ b/dovetail/tests/unit/test_parser.py
@@ -17,7 +17,7 @@ import unittest
import yaml
-import dovetail.parser as dovetail_parser
+import parser as dovetail_parser
class TestParser(unittest.TestCase):
@@ -27,6 +27,7 @@ class TestParser(unittest.TestCase):
def setUp(self):
"""Test case setup"""
logging.disable(logging.CRITICAL)
+ dovetail_parser.Parser.create_log()
def test_parser_cmd(self):
"""Test whether the command is correctly parsed."""
diff --git a/dovetail/utils/dovetail_logger.py b/dovetail/utils/dovetail_logger.py
index 6a2d38d5..8afa08a1 100644
--- a/dovetail/utils/dovetail_logger.py
+++ b/dovetail/utils/dovetail_logger.py
@@ -25,22 +25,12 @@ import logging
import os
from conf.dovetail_config import DovetailConfig as dt_config
-import dovetail_utils as dt_utils
-
-
-def clean_results_dir():
- result_path = dt_config.dovetail_config['result_dir']
- if os.path.exists(result_path):
- cmd = 'sudo rm -rf %s/*' % (result_path)
- dt_utils.exec_cmd(cmd, exit_on_error=False)
-
-clean_results_dir()
class Logger:
def __init__(self, logger_name):
- CI_DEBUG = os.getenv('CI_DEBUG')
+ DEBUG = os.getenv('DEBUG')
self.logger = logging.getLogger(logger_name)
self.logger.propagate = 0
@@ -50,7 +40,7 @@ class Logger:
formatter = logging.Formatter('%(asctime)s - %(name)s - '
'%(levelname)s - %(message)s')
ch.setFormatter(formatter)
- if CI_DEBUG is not None and CI_DEBUG.lower() == "true":
+ if DEBUG is not None and DEBUG.lower() == "true":
ch.setLevel(logging.DEBUG)
else:
ch.setLevel(logging.INFO)