diff options
Diffstat (limited to 'utils/test/reporting')
| -rw-r--r-- | utils/test/reporting/img/weather-clear.png | bin | 0 -> 1560 bytes | |||
| -rw-r--r-- | utils/test/reporting/img/weather-few-clouds.png | bin | 0 -> 1927 bytes | |||
| -rw-r--r-- | utils/test/reporting/img/weather-overcast.png | bin | 0 -> 1588 bytes | |||
| -rw-r--r-- | utils/test/reporting/img/weather-storm.png | bin | 0 -> 2137 bytes | |||
| -rw-r--r-- | utils/test/reporting/index-status-tmpl.html | 92 | ||||
| -rw-r--r-- | utils/test/reporting/index-tempest-tmpl.html | 2 | ||||
| -rw-r--r-- | utils/test/reporting/index-vims-tmpl.html | 12 | ||||
| -rw-r--r-- | utils/test/reporting/index.html | 52 | ||||
| -rw-r--r-- | utils/test/reporting/reporting-status.py | 178 |
9 files changed, 329 insertions, 7 deletions
diff --git a/utils/test/reporting/img/weather-clear.png b/utils/test/reporting/img/weather-clear.png Binary files differnew file mode 100644 index 000000000..a0d967750 --- /dev/null +++ b/utils/test/reporting/img/weather-clear.png diff --git a/utils/test/reporting/img/weather-few-clouds.png b/utils/test/reporting/img/weather-few-clouds.png Binary files differnew file mode 100644 index 000000000..acfa78398 --- /dev/null +++ b/utils/test/reporting/img/weather-few-clouds.png diff --git a/utils/test/reporting/img/weather-overcast.png b/utils/test/reporting/img/weather-overcast.png Binary files differnew file mode 100644 index 000000000..4296246d0 --- /dev/null +++ b/utils/test/reporting/img/weather-overcast.png diff --git a/utils/test/reporting/img/weather-storm.png b/utils/test/reporting/img/weather-storm.png Binary files differnew file mode 100644 index 000000000..956f0e20f --- /dev/null +++ b/utils/test/reporting/img/weather-storm.png diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/index-status-tmpl.html new file mode 100644 index 000000000..130ecd5c1 --- /dev/null +++ b/utils/test/reporting/index-status-tmpl.html @@ -0,0 +1,92 @@ + <html> + <head> + <meta charset="utf-8"> + <!-- Bootstrap core CSS --> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet"> + <link href="default.css" rel="stylesheet"> + <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script> + <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> + <script type="text/javascript"> + $(document).ready(function (){ + $(".btn-more").click(function() { + $(this).hide(); + $(this).parent().find(".panel-default").show(); + }); + }) + </script> + </head> + <body> + <div class="container"> + <div class="masthead"> + <h3 class="text-muted">Functest status page</h3> + <nav> + <ul class="nav nav-justified"> + <li class="active"><a href="index.html">Home</a></li> + <li><a href="index-status-apex.html">Apex</a></li> + <li><a href="index-status-compass.html">Compass</a></li> + <li><a href="index-status-fuel.html">Fuel</a></li> + <li><a href="index-status-joid.html">Joid</a></li> + </ul> + </nav> + </div> +<div class="row"> + <div class="col-md-1"></div> + <div class="col-md-10"> + <div class="page-header"> + <h2>{{installer}}</h2> + </div> + + <div class="scenario-overview"> + <div class="panel-heading"><h4><b>List of last scenarios run over the last 7 days </b></h4></div> + <table class="table"> + <tr> + <th width="80%">Scenario</th> + <th width="20%">Iteration</th> + </tr> + {% for scenario,iteration in scenario_stats.iteritems() -%} + <tr class="tr-ok"> + <td>{{scenario}}</td> + <td>{{iteration}}</td> + </tr> + {%- endfor %} + </table> + </div> + + + + {% for scenario, iteration in scenario_stats.iteritems() -%} + <div class="scenario-part"> + <div class="page-header"> + <h3><span class="glyphicon glyphicon-chevron-right"> <b>{{scenario}}</b></h3> + </div> + <div class="panel panel-default"> + <div class="panel-heading"> + <span class="panel-header-item"> + </span> + </div> + <table class="table"> + <tr> + {% for test in items[scenario] -%} + <th>{{test.getName() }}</th> + {%- endfor %} + </tr> + <tr class="tr-weather-weather"> + {% for test in items[scenario] -%} + {% if test.getCriteria() > 3 -%} + <td><img src="./img/weather-clear.png"></td> + {%- elif test.getCriteria() > 2 -%} + <td><img src="./img/weather-few-clouds.png"></td> + {%- elif test.getCriteria() > 1 -%} + <td><img src="./img/weather-overcast.png"></td> + {%- else -%} + <td><img src="./img/weather-storm.png"></td> + {%- endif %} + {%- endfor %} + </tr> + </table> + </div> + </div> + {%- endfor %} + </div> + <div class="col-md-1"></div> +</div> diff --git a/utils/test/reporting/index-tempest-tmpl.html b/utils/test/reporting/index-tempest-tmpl.html index 24d87bee2..be0b79734 100644 --- a/utils/test/reporting/index-tempest-tmpl.html +++ b/utils/test/reporting/index-tempest-tmpl.html @@ -21,7 +21,7 @@ <h3 class="text-muted">Tempest status page</h3> <nav> <ul class="nav nav-justified"> - <li class="active"><a href="#">Home</a></li> + <li class="active"><a href="index.html">Home</a></li> <li><a href="index-tempest-apex.html">Apex</a></li> <li><a href="index-tempest-compass.html">Compass</a></li> <li><a href="index-tempest-fuel.html">Fuel</a></li> diff --git a/utils/test/reporting/index-vims-tmpl.html b/utils/test/reporting/index-vims-tmpl.html index 4d1c50915..8858182c1 100644 --- a/utils/test/reporting/index-vims-tmpl.html +++ b/utils/test/reporting/index-vims-tmpl.html @@ -21,11 +21,11 @@ <h3 class="text-muted">vIMS status page</h3> <nav> <ul class="nav nav-justified"> - <li class="active"><a href="#">Home</a></li> - <li><a href="index-fuel.html">Fuel</a></li> - <li><a href="index-compass.html">Compass</a></li> - <li><a href="index-joid.html">JOID</a></li> - <li><a href="index-apex.html">APEX</a></li> + <li class="active"><a href="index.html">Home</a></li> + <li><a href="index-vims-fuel.html">Fuel</a></li> + <li><a href="index--vims-compass.html">Compass</a></li> + <li><a href="index-vims-joid.html">JOID</a></li> + <li><a href="index-vims-apex.html">APEX</a></li> </ul> </nav> </div> @@ -88,4 +88,4 @@ {%- endfor %} </div> <div class="col-md-1"></div> -</div>
\ No newline at end of file +</div> diff --git a/utils/test/reporting/index.html b/utils/test/reporting/index.html new file mode 100644 index 000000000..af4033567 --- /dev/null +++ b/utils/test/reporting/index.html @@ -0,0 +1,52 @@ + <html> + <head> + <meta charset="utf-8"> + <!-- Bootstrap core CSS --> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet"> + <link href="default.css" rel="stylesheet"> + <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script> + <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> + <script type="text/javascript"> + $(document).ready(function (){ + $(".btn-more").click(function() { + $(this).hide(); + $(this).parent().find(".panel-default").show(); + }); + }) + </script> + </head> + <body> + <div class="container"> + <div class="masthead"> + <h3 class="text-muted">Functest reporting page</h3> + <nav> + <ul class="nav nav-justified"> + <li class="active"><a href="#">Home</a></li> + <li><a href="./index-status-apex.html">Status</a></li> + <li><a href="./index-tempest-apex.html">Tempest</a></li> + <li><a href="./index-vims-apex.html">vIMS</a></li> + </ul> + </nav> + </div> +<div class="row"> + <div class="col-md-1"></div> + <div class="col-md-10"> + <div class="page-main"> + <h2>Functest</h2> + This project develops test suites that cover functionaling test cases in OPNFV. + <br>The test suites are integrated in the continuation integration (CI) framework and used to evaluate/validate scenario. + <br> Weekly meeting: every Tuesday 8 AM UTC + <br> IRC chan #opnfv-testperf + + <br> + <h2>Useful Links</h2> + <li><a href="http://events.linuxfoundation.org/sites/events/files/slides/Functest%20in%20Depth_0.pdf">Functest in Depth</a></li> + <li><a href="https://git.opnfv.org/cgit/functest">Functest Repo</a></li> + <li><a href="https://wiki.opnfv.org/opnfv_functional_testing">Functest Project</a></li> + <li><a href="https://build.opnfv.org/ci/view/functest/">Functest Jenkins page</a></li> + <li><a href="https://jira.opnfv.org/secure/RapidBoard.jspa?rapidView=59&projectKey=FUNCTEST">JIRA</a></li> + + </div> + </div> + <div class="col-md-1"></div> +</div> diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/reporting-status.py new file mode 100644 index 000000000..b27af4b14 --- /dev/null +++ b/utils/test/reporting/reporting-status.py @@ -0,0 +1,178 @@ +from urllib2 import Request, urlopen, URLError +import urllib2 +import json +import jinja2 +import os +import random + + +class TestCase(object): + def __init__(self, name, project, criteria=-1): + self.name = name + self.project = project + self.criteria = criteria + + def getName(self): + return self.name + + def getProject(self): + return self.project + + def getCriteria(self): + return self.criteria + + def setCriteria(self, criteria): + self.criteria = criteria + + +def getApiResults(case, installer): + case = case.getName() + + # to remove proxy (to be removed at the end for local test only) + # proxy_handler = urllib2.ProxyHandler({}) + # opener = urllib2.build_opener(proxy_handler) + # urllib2.install_opener(opener) + url = "http://testresults.opnfv.org/testapi/results?case=" + case + "&period=30&installer=" + installer + #url = "http://127.0.0.1:8000/results?case=" + case + "&period=30&installer=" + installer + request = Request(url) + + try: + response = urlopen(request) + k = response.read() + results = json.loads(k) + except URLError, e: + print 'No kittez. Got an error code:', e + + return results + + +def getScenarios(case, installer): + + results = getApiResults(case, installer) + test_results = results['test_results'] + + if test_results is not None: + test_results.reverse() + + scenario_results = {} + + for r in test_results: + # Retrieve all the scenarios per installer + if not r['version'] in scenario_results.keys(): + scenario_results[r['version']] = [] + scenario_results[r['version']].append(r) + + return scenario_results + + +def getScenarioStats(scenario_results): + scenario_stats = {} + for k, v in scenario_results.iteritems(): + scenario_stats[k] = len(v) + + return scenario_stats + + +def getResult(testCase, installer): + + # retrieve raw results + results = getApiResults(testCase, installer) + # let's concentrate on test results only + test_results = results['test_results'] + + # if results found, analyze them + if test_results is not None: + test_results.reverse() + + scenario_results = {} + + for r in test_results: + if not r['version'] in scenario_results.keys(): + scenario_results[r['version']] = [] + scenario_results[r['version']].append(r) + + for s, s_result in scenario_results.items(): + scenario_results[s] = s_result[0:5] + # For each scenario, we build a result object to deal with + # results, criteria and error handling + for result in scenario_results[s]: + result["creation_date"] = result["creation_date"].split(".")[0] + + # Cannot be fully generic + # need to look for specific criteria case by case + # TODO add a criteria passed/failed in DB?? + # TODO result["Success_criteria"] = result["success_criteria"] + # meanwhile just random.... + # and consider the last random arbitrarily + # 4 levels for the results + # 3: 4+ consecutive runs passing the success criteria + # 2: <4 successful consecutive runs but passing the criteria + # 1: close to pass the success criteria + # 0: 0% success, not passing + # + + return int(random.random()*4)+1 + +# ****************************************************************************** +# ****************************************************************************** +# ****************************************************************************** +# ****************************************************************************** +# ****************************************************************************** + +# as the criteria are all difference, we shall use a common way to indicate +# the criteria +# 100 = 100% = all the test must be OK +# 90 = 90% = all the test must be above 90% of success rate +# TODO harmonize success criteria +# some criteria could be the duration, the success rate, the packet loss,... +# to be done case by case +# TODo create TestCriteria Object + + +installers = ["apex", "compass", "fuel", "joid"] +# init just tempest to get the scenario as all the scenarios run Temepst +tempest = TestCase("Tempest", "functest", -1) + +for installer in installers: + + scenario_results = getScenarios(tempest, installer) + scenario_stats = getScenarioStats(scenario_results) + + items = {} + + for s, s_result in scenario_results.items(): + + vPing = TestCase("vPing", "functest") + vPing_userdata = TestCase("vPing_userdata", "functest") + tempest = TestCase("Tempest", "functest") + rally = TestCase("Rally", "functest") + odl = TestCase("ODL", "functest") + onos = TestCase("ONOS", "functest") + ovno = TestCase("OVNO", "functest") + vIMS = TestCase("vIMS", "functest") + doctor = TestCase("doctor-notification", "doctor") + promise = TestCase("promise", "promise") + odl_vpn = TestCase("ODL VPN Service tests", "sdnvpn") + bgpvpn_api = TestCase("OpenStack Neutron BGPVPN API extension tests", + "sdnvpn") + testCases = [vPing, vPing_userdata, tempest, rally, odl, onos, vIMS, + doctor, promise] + + for testCase in testCases: + result = getResult(testCase, installer) + testCase.setCriteria(result) + # print "case %s (%s) = %s " % (testCase.getName(), s, result) + items[s] = testCases + + templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + templateEnv = jinja2.Environment(loader=templateLoader) + + TEMPLATE_FILE = "index-status-tmpl.html" + template = templateEnv.get_template(TEMPLATE_FILE) + + outputText = template.render(scenario_stats=scenario_stats, + items=items, + installer=installer) + + with open("index-status-" + installer + ".html", "wb") as fh: + fh.write(outputText) |
