Functest reporting refactoring

integrate the notion of version (brahmaputra, master, ..)
change dir structure

Change-Id: Ieb8aed811ed4e1ab8738fb02e4db411da3d07ca2
Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>

12 files changed, 84 insertions, 83 deletions

diff --git a/utils/test/reporting/default.css b/utils/test/reporting/functest/default.css
index 0e330e965..0e330e965 100644
--- a/utils/test/reporting/default.css
+++ b/utils/test/reporting/functest/default.css
diff --git a/utils/test/reporting/img/weather-clear.png b/utils/test/reporting/functest/img/weather-clear.png
index a0d967750..a0d967750 100644
--- a/utils/test/reporting/img/weather-clear.png
+++ b/utils/test/reporting/functest/img/weather-clear.png
diff --git a/utils/test/reporting/img/weather-few-clouds.png b/utils/test/reporting/functest/img/weather-few-clouds.png
index acfa78398..acfa78398 100644
--- a/utils/test/reporting/img/weather-few-clouds.png
+++ b/utils/test/reporting/functest/img/weather-few-clouds.png
diff --git a/utils/test/reporting/img/weather-overcast.png b/utils/test/reporting/functest/img/weather-overcast.png
index 4296246d0..4296246d0 100644
--- a/utils/test/reporting/img/weather-overcast.png
+++ b/utils/test/reporting/functest/img/weather-overcast.png
diff --git a/utils/test/reporting/img/weather-storm.png b/utils/test/reporting/functest/img/weather-storm.png
index 956f0e20f..956f0e20f 100644
--- a/utils/test/reporting/img/weather-storm.png
+++ b/utils/test/reporting/functest/img/weather-storm.png
diff --git a/utils/test/reporting/index.html b/utils/test/reporting/functest/index.html
index af4033567..af4033567 100644
--- a/utils/test/reporting/index.html
+++ b/utils/test/reporting/functest/index.html
diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/functest/reporting-status.py
index e15bac9f7..9e6aeb1ad 100644
--- a/utils/test/reporting/reporting-status.py
+++ b/utils/test/reporting/functest/reporting-status.py
@@ -4,6 +4,7 @@ import jinja2
 import os
 import re
 import requests
+import sys
 import time
 import yaml
 
@@ -12,11 +13,12 @@ functest_test_list = ['vPing', 'vPing_userdata',
                       'Tempest', 'Rally',
                       'ODL', 'ONOS', 'vIMS']
 # functest_test_list = ['vPing']
-# functest_test_list = []
 companion_test_list = ['doctor/doctor-notification', 'promise/promise']
 # companion_test_list = []
 installers = ["apex", "compass", "fuel", "joid"]
-# installers = ["apex"]
+# installers = ["fuel"]
+versions = ["brahmaputra", "master"]
+# versions = ["master"]
 PERIOD = 10
 
 # Correspondance between the name of the test case and the name in the DB
@@ -100,7 +102,7 @@ class TestCase(object):
         self.isRunnable = is_runnable
 
 
-def getApiResults(case, installer, scenario):
+def getApiResults(case, installer, scenario, version):
     case = case.getName()
     results = json.dumps([])
     # to remove proxy (to be removed at the end for local test only)
@@ -111,7 +113,7 @@ def getApiResults(case, installer, scenario):
     #       "&period=30&installer=" + installer
     url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
           "&period=" + str(PERIOD) + "&installer=" + installer + \
-          "&scenario=" + scenario
+          "&scenario=" + scenario + "&version=" + version
     request = Request(url)
 
     try:
@@ -124,11 +126,12 @@ def getApiResults(case, installer, scenario):
     return results
 
 
-def getScenarios(case, installer):
+def getScenarios(case, installer, version):
 
     case = case.getName()
     url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
-          "&period=" + str(PERIOD) + "&installer=" + installer
+          "&period=" + str(PERIOD) + "&installer=" + installer + \
+          "&version=" + version
     request = Request(url)
 
     try:
@@ -147,9 +150,9 @@ def getScenarios(case, installer):
 
         for r in test_results:
             # Retrieve all the scenarios per installer
-            if not r['version'] in scenario_results.keys():
-                scenario_results[r['version']] = []
-            scenario_results[r['version']].append(r)
+            if not r['scenario'] in scenario_results.keys():
+                scenario_results[r['scenario']] = []
+            scenario_results[r['scenario']].append(r)
 
     return scenario_results
 
@@ -174,10 +177,10 @@ def getNbtestOk(results):
     return nb_test_ok
 
 
-def getResult(testCase, installer, scenario):
+def getResult(testCase, installer, scenario, version):
 
     # retrieve raw results
-    results = getApiResults(testCase, installer, scenario)
+    results = getApiResults(testCase, installer, scenario, version)
     # let's concentrate on test results only
     test_results = results['test_results']
 
@@ -230,16 +233,6 @@ def getResult(testCase, installer, scenario):
 # ******************************************************************************
 # ******************************************************************************
 
-# as the criteria are all difference, we shall use a common way to indicate
-# the criteria
-# 100 = 100% = all the test must be OK
-# 90 = 90% = all the test must be above 90% of success rate
-# TODO harmonize success criteria
-# some criteria could be the duration, the success rate, the packet loss,...
-# to be done case by case
-# TODo create TestCriteria Object
-
-
 # init just tempest to get the list of scenarios
 # as all the scenarios run Tempest
 tempest = TestCase("Tempest", "functest", -1)
@@ -252,55 +245,63 @@ functest_yaml_config = yaml.load(response.text)
 print "****************************************"
 print "*   Generating reporting.....          *"
 print "****************************************"
-# For all the installers
-for installer in installers:
-    # get scenarios
-    scenario_results = getScenarios(tempest, installer)
-    scenario_stats = getScenarioStats(scenario_results)
-
-    items = {}
-    # For all the scenarios get results
-    for s, s_result in scenario_results.items():
-        testCases = []
-        # For each scenario declare the test cases
-        # Functest cases
-        for test_case in functest_test_list:
-            testCases.append(TestCase(test_case, "functest"))
-
-        # project/case
-        for test_case in companion_test_list:
-            test_split = test_case.split("/")
-            test_project = test_split[0]
-            test_case = test_split[1]
-            testCases.append(TestCase(test_case, test_project))
-
-        # Check if test case is runnable according to the installer, scenario
-        for test_case in testCases:
-            test_case.checkRunnable(installer, s, functest_yaml_config)
-            # print "testcase %s is %s" % (test_case.getName(),
-            #                              test_case.isRunnable)
-
-        print "--------------------------"
-        print "%s / %s:" % (installer, s)
-        for testCase in testCases:
-            time.sleep(1)
-            if testCase.isRunnable:
-                print "    Searching results for case %s " % testCase.getName()
-                result = getResult(testCase, installer, s)
-                testCase.setCriteria(result)
-                items[s] = testCases
-        print "--------------------------"
-    print "****************************************"
-    templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
-    templateEnv = jinja2.Environment(loader=templateLoader)
-
-    TEMPLATE_FILE = "index-status-tmpl.html"
-    template = templateEnv.get_template(TEMPLATE_FILE)
-
-    outputText = template.render(scenario_stats=scenario_stats,
-                                 items=items,
-                                 installer=installer,
-                                 period=PERIOD)
-
-    with open("index-status-" + installer + ".html", "wb") as fh:
-        fh.write(outputText)
+# For all the versions
+for version in versions:
+    # For all the installers
+    for installer in installers:
+        # get scenarios
+        scenario_results = getScenarios(tempest, installer, version)
+        scenario_stats = getScenarioStats(scenario_results)
+
+        items = {}
+        # For all the scenarios get results
+        for s, s_result in scenario_results.items():
+            testCases = []
+            # For each scenario declare the test cases
+            # Functest cases
+            for test_case in functest_test_list:
+                testCases.append(TestCase(test_case, "functest"))
+
+            # project/case
+            for test_case in companion_test_list:
+                test_split = test_case.split("/")
+                test_project = test_split[0]
+                test_case = test_split[1]
+                testCases.append(TestCase(test_case, test_project))
+
+            # Check if test case is runnable / installer, scenario
+            try:
+                for test_case in testCases:
+                    test_case.checkRunnable(installer, s, functest_yaml_config)
+                    # print "testcase %s is %s" % (test_case.getName(),
+                    #                              test_case.isRunnable)
+                print "--------------------------"
+                print "installer %s, version %s, scenario %s:" % (installer, version, s)
+                for testCase in testCases:
+                    time.sleep(1)
+                    if testCase.isRunnable:
+                        print " Searching results for case %s " % (testCase.getName())
+                        result = getResult(testCase, installer, s, version)
+                        testCase.setCriteria(result)
+                        items[s] = testCases
+                print "--------------------------"
+            except:
+                print "installer %s, version %s, scenario %s" % (installer, version, s)
+                print "No data available , error %s " % (sys.exc_info()[0])
+
+        print "****************************************"
+        templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
+        templateEnv = jinja2.Environment(loader=templateLoader)
+
+        TEMPLATE_FILE = "./template/index-status-tmpl.html"
+        template = templateEnv.get_template(TEMPLATE_FILE)
+
+        outputText = template.render(scenario_stats=scenario_stats,
+                                     items=items,
+                                     installer=installer,
+                                     period=PERIOD,
+                                     version=version)
+
+        with open("./release/" + version +
+                  "/index-status-" + installer + ".html", "wb") as fh:
+            fh.write(outputText)
diff --git a/utils/test/reporting/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py
index 944b42809..563e53010 100644
--- a/utils/test/reporting/reporting-tempest.py
+++ b/utils/test/reporting/functest/reporting-tempest.py
@@ -88,12 +88,12 @@ for installer in installers:
     templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
     templateEnv = jinja2.Environment(loader=templateLoader)
 
-    TEMPLATE_FILE = "index-tempest-tmpl.html"
+    TEMPLATE_FILE = "./template/index-tempest-tmpl.html"
     template = templateEnv.get_template(TEMPLATE_FILE)
 
     outputText = template.render(scenario_results=scenario_results,
                                  items=items,
                                  installer=installer)
 
-    with open("index-tempest-" + installer + ".html", "wb") as fh:
+    with open("./release/index-tempest-" + installer + ".html", "wb") as fh:
         fh.write(outputText)
diff --git a/utils/test/reporting/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py
index cf43f3ebc..78ca9f5b3 100644
--- a/utils/test/reporting/reporting-vims.py
+++ b/utils/test/reporting/functest/reporting-vims.py
@@ -72,12 +72,12 @@ for installer in installers:
     templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
     templateEnv = jinja2.Environment( loader=templateLoader )
 
-    TEMPLATE_FILE = "index-vims-tmpl.html"
+    TEMPLATE_FILE = "./template/index-vims-tmpl.html"
     template = templateEnv.get_template( TEMPLATE_FILE )
 
     outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer)
 
-    with open("index-vims" + installer + ".html", "wb") as fh:
+    with open("./release/index-vims" + installer + ".html", "wb") as fh:
         fh.write(outputText)
 
 
diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html
index 0fd470d6f..604f2c8e4 100644
--- a/utils/test/reporting/index-status-tmpl.html
+++ b/utils/test/reporting/functest/template/index-status-tmpl.html
@@ -18,7 +18,7 @@
     <body>
     <div class="container">
       <div class="masthead">
-        <h3 class="text-muted">Functest status page</h3>
+        <h3 class="text-muted">Functest status page ({{version}})</h3>
         <nav>
           <ul class="nav nav-justified">
             <li class="active"><a href="index.html">Home</a></li>
@@ -37,7 +37,7 @@
         </div>
 
         <div class="scenario-overview">
-            <div class="panel-heading"><h4><b>List of last scenarios run over the last {{period}} days </b></h4></div>
+            <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div>
                 <table class="table">
                     <tr>
                         <th width="80%">Scenario</th>
@@ -75,13 +75,13 @@
                             {% if test.isRunnable is sameas false -%}
                                 <td>N.R</td>
                             {% elif test.getCriteria() > 2 -%}
-                                <td><img src="./img/weather-clear.png"></td>
+                                <td><img src="../../img/weather-clear.png"></td>
                             {%- elif test.getCriteria() > 1 -%}
-                                <td><img src="./img/weather-few-clouds.png"></td>
+                                <td><img src="../../img/weather-few-clouds.png"></td>
                             {%- elif test.getCriteria() > 0 -%}
-                                <td><img src="./img/weather-overcast.png"></td>
+                                <td><img src="../../img/weather-overcast.png"></td>
                             {%- else -%}
-                                <td><img src="./img/weather-storm.png"></td>
+                                <td><img src="../../img/weather-storm.png"></td>
                             {%- endif %}
                             {%- endfor %}
                         </tr>
diff --git a/utils/test/reporting/index-tempest-tmpl.html b/utils/test/reporting/functest/template/index-tempest-tmpl.html
index be0b79734..be0b79734 100644
--- a/utils/test/reporting/index-tempest-tmpl.html
+++ b/utils/test/reporting/functest/template/index-tempest-tmpl.html
diff --git a/utils/test/reporting/index-vims-tmpl.html b/utils/test/reporting/functest/template/index-vims-tmpl.html
index 8858182c1..8858182c1 100644
--- a/utils/test/reporting/index-vims-tmpl.html
+++ b/utils/test/reporting/functest/template/index-vims-tmpl.html