diff options
| -rw-r--r-- | utils/test/reporting/index-status-tmpl.html | 10 | ||||
| -rw-r--r-- | utils/test/reporting/reporting-status.py | 256 |
2 files changed, 198 insertions, 68 deletions
diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/index-status-tmpl.html index 130ecd5c1..0fd470d6f 100644 --- a/utils/test/reporting/index-status-tmpl.html +++ b/utils/test/reporting/index-status-tmpl.html @@ -37,7 +37,7 @@ </div> <div class="scenario-overview"> - <div class="panel-heading"><h4><b>List of last scenarios run over the last 7 days </b></h4></div> + <div class="panel-heading"><h4><b>List of last scenarios run over the last {{period}} days </b></h4></div> <table class="table"> <tr> <th width="80%">Scenario</th> @@ -72,11 +72,13 @@ </tr> <tr class="tr-weather-weather"> {% for test in items[scenario] -%} - {% if test.getCriteria() > 3 -%} + {% if test.isRunnable is sameas false -%} + <td>N.R</td> + {% elif test.getCriteria() > 2 -%} <td><img src="./img/weather-clear.png"></td> - {%- elif test.getCriteria() > 2 -%} - <td><img src="./img/weather-few-clouds.png"></td> {%- elif test.getCriteria() > 1 -%} + <td><img src="./img/weather-few-clouds.png"></td> + {%- elif test.getCriteria() > 0 -%} <td><img src="./img/weather-overcast.png"></td> {%- else -%} <td><img src="./img/weather-storm.png"></td> diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/reporting-status.py index b27af4b14..e15bac9f7 100644 --- a/utils/test/reporting/reporting-status.py +++ b/utils/test/reporting/reporting-status.py @@ -1,16 +1,45 @@ from urllib2 import Request, urlopen, URLError -import urllib2 import json import jinja2 import os -import random +import re +import requests +import time +import yaml + +# Declaration of the variables +functest_test_list = ['vPing', 'vPing_userdata', + 'Tempest', 'Rally', + 'ODL', 'ONOS', 'vIMS'] +# functest_test_list = ['vPing'] +# functest_test_list = [] +companion_test_list = ['doctor/doctor-notification', 'promise/promise'] +# companion_test_list = [] +installers = ["apex", "compass", "fuel", "joid"] +# installers = ["apex"] +PERIOD = 10 + +# Correspondance between the name of the test case and the name in the DB +# ideally we should modify the DB to avoid such interface.... +# '<name in the DB':'<name in the config'> +# I know it is uggly... +test_match_matrix = {'vPing': 'vping_ssh', + 'vPing_userdata': 'vping_userdata', + 'ODL': 'odl', + 'ONOS': 'onos', + 'Tempest': 'tempest', + 'Rally': 'rally', + 'vIMS': 'vims', + 'doctor-notification': 'doctor', + 'promise': 'promise'} class TestCase(object): - def __init__(self, name, project, criteria=-1): + def __init__(self, name, project, criteria=-1, isRunnable=True): self.name = name self.project = project self.criteria = criteria + self.isRunnable = isRunnable def getName(self): return self.name @@ -24,16 +53,65 @@ class TestCase(object): def setCriteria(self, criteria): self.criteria = criteria - -def getApiResults(case, installer): + def setIsRunnable(self, isRunnable): + self.isRunnable = isRunnable + + def checkRunnable(self, installer, scenario, config): + # Re-use Functest declaration + # Retrieve Functest configuration file functest_config.yaml + is_runnable = True + config_test = "" + TEST_ENV = functest_yaml_config.get("test-dependencies") + + # print " *********************** " + # print TEST_ENV + # print " ---------------------- " + # print "case = " + self.name + # print "installer = " + installer + # print "scenario = " + scenario + # print "project = " + self.project + + # Retrieve test constraints + case_name_formated = test_match_matrix[self.name] + + try: + config_test = TEST_ENV[self.project][case_name_formated] + except KeyError: + # if not defined in dependencies => no dependencies + config_test = TEST_ENV[case_name_formated] + except Exception, e: + print "Error [getTestEnv]:", e + + # Retrieve test execution param + test_execution_context = {"installer": installer, + "scenario": scenario} + # By default we assume that all the tests are always runnable... + # if test_env not empty => dependencies to be checked + if config_test is not None and len(config_test) > 0: + # possible criteria = ["installer", "scenario"] + # consider test criteria from config file + # compare towards CI env through CI en variable + for criteria in config_test: + if re.search(config_test[criteria], + test_execution_context[criteria]) is None: + # print "Test "+ test + " cannot be run on the environment" + is_runnable = False + # print is_runnable + self.isRunnable = is_runnable + + +def getApiResults(case, installer, scenario): case = case.getName() - + results = json.dumps([]) # to remove proxy (to be removed at the end for local test only) # proxy_handler = urllib2.ProxyHandler({}) # opener = urllib2.build_opener(proxy_handler) # urllib2.install_opener(opener) - url = "http://testresults.opnfv.org/testapi/results?case=" + case + "&period=30&installer=" + installer - #url = "http://127.0.0.1:8000/results?case=" + case + "&period=30&installer=" + installer + # url = "http://127.0.0.1:8000/results?case=" + case + \ + # "&period=30&installer=" + installer + url = "http://testresults.opnfv.org/testapi/results?case=" + case + \ + "&period=" + str(PERIOD) + "&installer=" + installer + \ + "&scenario=" + scenario request = Request(url) try: @@ -48,7 +126,18 @@ def getApiResults(case, installer): def getScenarios(case, installer): - results = getApiResults(case, installer) + case = case.getName() + url = "http://testresults.opnfv.org/testapi/results?case=" + case + \ + "&period=" + str(PERIOD) + "&installer=" + installer + request = Request(url) + + try: + response = urlopen(request) + k = response.read() + results = json.loads(k) + except URLError, e: + print 'Got an error code:', e + test_results = results['test_results'] if test_results is not None: @@ -73,10 +162,22 @@ def getScenarioStats(scenario_results): return scenario_stats -def getResult(testCase, installer): +def getNbtestOk(results): + nb_test_ok = 0 + for r in results: + for k, v in r.iteritems(): + try: + if "passed" in v: + nb_test_ok += 1 + except: + print "Cannot retrieve test status" + return nb_test_ok + + +def getResult(testCase, installer, scenario): # retrieve raw results - results = getApiResults(testCase, installer) + results = getApiResults(testCase, installer, scenario) # let's concentrate on test results only test_results = results['test_results'] @@ -84,34 +185,44 @@ def getResult(testCase, installer): if test_results is not None: test_results.reverse() - scenario_results = {} + scenario_results = [] - for r in test_results: - if not r['version'] in scenario_results.keys(): - scenario_results[r['version']] = [] - scenario_results[r['version']].append(r) + # print " ---------------- " + # print test_results + # print " ---------------- " + # print "nb of results:" + str(len(test_results)) - for s, s_result in scenario_results.items(): - scenario_results[s] = s_result[0:5] - # For each scenario, we build a result object to deal with - # results, criteria and error handling - for result in scenario_results[s]: - result["creation_date"] = result["creation_date"].split(".")[0] - - # Cannot be fully generic - # need to look for specific criteria case by case - # TODO add a criteria passed/failed in DB?? - # TODO result["Success_criteria"] = result["success_criteria"] - # meanwhile just random.... - # and consider the last random arbitrarily - # 4 levels for the results - # 3: 4+ consecutive runs passing the success criteria - # 2: <4 successful consecutive runs but passing the criteria - # 1: close to pass the success criteria - # 0: 0% success, not passing - # - - return int(random.random()*4)+1 + for r in test_results: + # print r["creation_date"] + # print r["criteria"] + scenario_results.append({r["creation_date"]: r["criteria"]}) + # sort results + scenario_results.sort() + # 4 levels for the results + # 3: 4+ consecutive runs passing the success criteria + # 2: <4 successful consecutive runs but passing the criteria + # 1: close to pass the success criteria + # 0: 0% success, not passing + test_result_indicator = 0 + nbTestOk = getNbtestOk(scenario_results) + # print "Nb test OK:"+ str(nbTestOk) + # check that we have at least 4 runs + if nbTestOk < 1: + test_result_indicator = 0 + elif nbTestOk < 2: + test_result_indicator = 1 + else: + # Test the last 4 run + if (len(scenario_results) > 3): + last4runResults = scenario_results[-4:] + if getNbtestOk(last4runResults): + test_result_indicator = 3 + else: + test_result_indicator = 2 + else: + test_result_indicator = 2 + print " >>>> Test indicator:" + str(test_result_indicator) + return test_result_indicator # ****************************************************************************** # ****************************************************************************** @@ -129,41 +240,57 @@ def getResult(testCase, installer): # TODo create TestCriteria Object -installers = ["apex", "compass", "fuel", "joid"] -# init just tempest to get the scenario as all the scenarios run Temepst +# init just tempest to get the list of scenarios +# as all the scenarios run Tempest tempest = TestCase("Tempest", "functest", -1) -for installer in installers: +# Retrieve the Functest configuration to detect which tests are relevant +# according to the installer, scenario +response = requests.get('https://git.opnfv.org/cgit/functest/plain/testcases/config_functest.yaml') +functest_yaml_config = yaml.load(response.text) +print "****************************************" +print "* Generating reporting..... *" +print "****************************************" +# For all the installers +for installer in installers: + # get scenarios scenario_results = getScenarios(tempest, installer) scenario_stats = getScenarioStats(scenario_results) items = {} - + # For all the scenarios get results for s, s_result in scenario_results.items(): - - vPing = TestCase("vPing", "functest") - vPing_userdata = TestCase("vPing_userdata", "functest") - tempest = TestCase("Tempest", "functest") - rally = TestCase("Rally", "functest") - odl = TestCase("ODL", "functest") - onos = TestCase("ONOS", "functest") - ovno = TestCase("OVNO", "functest") - vIMS = TestCase("vIMS", "functest") - doctor = TestCase("doctor-notification", "doctor") - promise = TestCase("promise", "promise") - odl_vpn = TestCase("ODL VPN Service tests", "sdnvpn") - bgpvpn_api = TestCase("OpenStack Neutron BGPVPN API extension tests", - "sdnvpn") - testCases = [vPing, vPing_userdata, tempest, rally, odl, onos, vIMS, - doctor, promise] - + testCases = [] + # For each scenario declare the test cases + # Functest cases + for test_case in functest_test_list: + testCases.append(TestCase(test_case, "functest")) + + # project/case + for test_case in companion_test_list: + test_split = test_case.split("/") + test_project = test_split[0] + test_case = test_split[1] + testCases.append(TestCase(test_case, test_project)) + + # Check if test case is runnable according to the installer, scenario + for test_case in testCases: + test_case.checkRunnable(installer, s, functest_yaml_config) + # print "testcase %s is %s" % (test_case.getName(), + # test_case.isRunnable) + + print "--------------------------" + print "%s / %s:" % (installer, s) for testCase in testCases: - result = getResult(testCase, installer) - testCase.setCriteria(result) - # print "case %s (%s) = %s " % (testCase.getName(), s, result) - items[s] = testCases - + time.sleep(1) + if testCase.isRunnable: + print " Searching results for case %s " % testCase.getName() + result = getResult(testCase, installer, s) + testCase.setCriteria(result) + items[s] = testCases + print "--------------------------" + print "****************************************" templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) templateEnv = jinja2.Environment(loader=templateLoader) @@ -172,7 +299,8 @@ for installer in installers: outputText = template.render(scenario_stats=scenario_stats, items=items, - installer=installer) + installer=installer, + period=PERIOD) with open("index-status-" + installer + ".html", "wb") as fh: fh.write(outputText) |