Merge "Add d3 graph presentation to yardstick reporting"

4 files changed, 111 insertions, 30 deletions

diff --git a/utils/test/reporting/yardstick/reporting-status.py b/utils/test/reporting/yardstick/reporting-status.py
index 60f1523bb..49809e9d8 100644
--- a/utils/test/reporting/yardstick/reporting-status.py
+++ b/utils/test/reporting/yardstick/reporting-status.py
@@ -8,10 +8,7 @@
 #
 import datetime
 import jinja2
-import requests
-import sys
-import time
-import yaml
+import os
 
 import reportingUtils as utils
 import reportingConf as conf
@@ -20,6 +17,7 @@ from scenarios import config as cf
 
 # Logger
 logger = utils.getLogger("Yardstick-Status")
+reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
 
 logger.info("*******************************************")
 logger.info("*   Generating reporting scenario status  *")
@@ -35,21 +33,23 @@ for version in conf.versions:
         # get scenarios results data
         scenario_results = utils.getScenarioStatus(installer, version)
         if 'colorado' == version:
-            stable_result = utils.getScenarioStatus(installer, 'stable/colorado')
-            for k,v in stable_result.items():
-                if not scenario_results.has_key(k):
+            stable_result = utils.getScenarioStatus(installer,
+                                                    'stable/colorado')
+            for k, v in stable_result.items():
+                if k not in scenario_results.keys():
                     scenario_results[k] = []
                 scenario_results[k] += stable_result[k]
         scenario_result_criteria = {}
 
         for s in scenario_results.keys():
-            if cf.has_key(installer) and cf[installer].has_key(s):
+            if installer in cf.keys() and s in cf[installer].keys():
                 scenario_results.pop(s)
 
         # From each scenarios get results list
         for s, s_result in scenario_results.items():
             logger.info("---------------------------------")
-            logger.info("installer %s, version %s, scenario %s:" % (installer, version, s))
+            logger.info("installer %s, version %s, scenario %s:" % (installer,
+                                                                    version, s))
 
             ten_criteria = len(s_result)
             ten_score = 0
@@ -62,15 +62,38 @@ for version in conf.versions:
             for v in four_result:
                 four_score += v
 
-            s_status = str(utils.get_status(four_result, s_result))
+            s_status = str(utils.get_percent(four_result, s_result))
             s_four_score = str(four_score) + '/' + str(four_criteria)
             s_ten_score = str(ten_score) + '/' + str(ten_criteria)
-            scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_four_score, s_ten_score)
+            s_score_percent = utils.get_percent(four_result, s_result)
 
             if '100' == s_status:
                 logger.info(">>>>> scenario OK, save the information")
             else:
-                logger.info(">>>> scenario not OK, last 4 iterations = %s, last 10 days = %s" % (s_four_score, s_ten_score))
+                logger.info(">>>> scenario not OK, last 4 iterations = %s, \
+                            last 10 days = %s" % (s_four_score, s_ten_score))
+
+            # Save daily results in a file
+            path_validation_file = (conf.REPORTING_PATH +
+                                    "/release/" + version +
+                                    "/scenario_history.txt")
+
+            if not os.path.exists(path_validation_file):
+                with open(path_validation_file, 'w') as f:
+                    info = 'date,scenario,installer,details,score\n'
+                    f.write(info)
+
+            with open(path_validation_file, "a") as f:
+                info = (reportingDate + "," + s + "," + installer +
+                        "," + s_ten_score + "," +
+                        str(s_score_percent) + "\n")
+                f.write(info)
+
+            scenario_result_criteria[s] = sr.ScenarioResult(s_status,
+                                                            s_four_score,
+                                                            s_ten_score,
+                                                            s_score_percent)
+
             logger.info("--------------------------")
 
         templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
@@ -82,7 +105,8 @@ for version in conf.versions:
         outputText = template.render(scenario_results=scenario_result_criteria,
                                      installer=installer,
                                      period=conf.PERIOD,
-                                     version=version)
+                                     version=version,
+                                     date=reportingDate)
 
         with open(conf.REPORTING_PATH + "/release/" + version +
                   "/index-status-" + installer + ".html", "wb") as fh:
diff --git a/utils/test/reporting/yardstick/reportingUtils.py b/utils/test/reporting/yardstick/reportingUtils.py
index 71eb9196c..ec9ed76dc 100644
--- a/utils/test/reporting/yardstick/reportingUtils.py
+++ b/utils/test/reporting/yardstick/reportingUtils.py
@@ -32,7 +32,7 @@ def getLogger(module):
 def getScenarioStatus(installer, version):
     url = (conf.URL_BASE + "?case=" + "scenario_status" +
            "&installer=" + installer +
-           "&version=" + version +"&period=" + str(conf.PERIOD))
+           "&version=" + version + "&period=" + str(conf.PERIOD))
     request = Request(url)
 
     try:
@@ -53,7 +53,7 @@ def getScenarioStatus(installer, version):
                     scenario_results[r['scenario']] = []
                 scenario_results[r['scenario']].append(r)
 
-        for k,v in scenario_results.items():
+        for k, v in scenario_results.items():
             # scenario_results[k] = v[:conf.LASTEST_TESTS]
             s_list = []
             for element in v:
@@ -66,20 +66,25 @@ def getScenarioStatus(installer, version):
     # return scenario_results
     return result_dict
 
+
 def subfind(given_list, pattern_list):
+
     for i in range(len(given_list)):
-        if given_list[i] == pattern_list[0] and given_list[i:i + conf.LASTEST_TESTS] == pattern_list:
+        if given_list[i] == pattern_list[0] and \
+                given_list[i:i + conf.LASTEST_TESTS] == pattern_list:
             return True
     return False
 
-def get_percent(status):
-    
+
+def _get_percent(status):
+
     if status * 100 % 6:
         return round(float(status) * 100 / 6, 1)
     else:
         return status * 100 / 6
 
-def get_status(four_list, ten_list):
+
+def get_percent(four_list, ten_list):
     four_score = 0
     ten_score = 0
 
@@ -97,13 +102,13 @@ def get_status(four_list, ten_list):
     else:
         status = four_score + 1
 
-    return get_percent(status)
+    return _get_percent(status)
 
 
 def _test():
     status = getScenarioStatus("compass", "master")
     print "status:++++++++++++++++++++++++"
-    print json.dumps(status,indent=4)
+    print json.dumps(status, indent=4)
 
 
 if __name__ == '__main__':    # pragma: no cover
diff --git a/utils/test/reporting/yardstick/scenarioResult.py b/utils/test/reporting/yardstick/scenarioResult.py
index 61ffb2ce7..1f7eb2b24 100644
--- a/utils/test/reporting/yardstick/scenarioResult.py
+++ b/utils/test/reporting/yardstick/scenarioResult.py
@@ -9,10 +9,12 @@
 
 
 class ScenarioResult(object):
-    def __init__(self, status, four_days_score='', ten_days_score=''):
+    def __init__(self, status, four_days_score='', ten_days_score='',
+                 score_percent=0.0):
         self.status = status
         self.four_days_score = four_days_score
         self.ten_days_score = ten_days_score
+        self.score_percent = score_percent
 
     def getStatus(self):
         return self.status
@@ -22,3 +24,6 @@ class ScenarioResult(object):
 
     def getFourDaysScore(self):
         return self.four_days_score
+
+    def getScorePercent(self):
+        return self.score_percent
diff --git a/utils/test/reporting/yardstick/template/index-status-tmpl.html b/utils/test/reporting/yardstick/template/index-status-tmpl.html
index 602ce8a74..5a4dc347c 100644
--- a/utils/test/reporting/yardstick/template/index-status-tmpl.html
+++ b/utils/test/reporting/yardstick/template/index-status-tmpl.html
@@ -3,9 +3,56 @@
     <meta charset="utf-8">
     <!-- Bootstrap core CSS -->
     <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet">
-    <link href="default.css" rel="stylesheet">
+    <link href="../../../css/default.css" rel="stylesheet">
     <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script>
     <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="http://d3js.org/d3.v2.min.js"></script>
+    <script type="text/javascript" src="../../../js/gauge.js"></script>
+    <script type="text/javascript" src="../../../js/trend.js"></script>
+    <script>
+        function onDocumentReady() {
+            // Gauge management
+            {% for scenario in scenario_results.keys() -%}
+            var gaugeScenario{{loop.index}} = gauge('#gaugeScenario{{loop.index}}');
+            {%- endfor %}
+            // assign success rate to the gauge
+            function updateReadings() {
+                {% for scenario in scenario_results.keys() -%}
+                 gaugeScenario{{loop.index}}.update({{scenario_results[scenario].getScorePercent()}});
+                 {%- endfor %}
+            }
+            updateReadings();
+        }
+
+        // trend line management
+        //d3.csv("./scenario_history.txt", function(data) {
+        d3.csv("./scenario_history.txt", function(data) {
+            // ***************************************
+            // Create the trend line
+            {% for scenario in scenario_results.keys() -%}
+            // for scenario {{scenario}}
+            // Filter results
+                var trend{{loop.index}} = data.filter(function(row) {
+                    return row["scenario"]=="{{scenario}}" && row["installer"]=="{{installer}}";
+                })
+            // Parse the date
+            trend{{loop.index}}.forEach(function(d) {
+                d.date = parseDate(d.date);
+                d.score = +d.score
+            });
+            // Draw the trend line
+            var mytrend = trend("#trend_svg{{loop.index}}",trend{{loop.index}})
+            // ****************************************
+            {%- endfor %}
+        });
+        if ( !window.isLoaded ) {
+            window.addEventListener("load", function() {
+            onDocumentReady();
+            }, false);
+        } else {
+            onDocumentReady();
+        }
+    </script>
     <script type="text/javascript">
     $(document).ready(function (){
         $(".btn-more").click(function() {
@@ -18,10 +65,10 @@
     <body>
     <div class="container">
       <div class="masthead">
-        <h3 class="text-muted">Yardstick status page ({{version}})</h3>
+          <h3 class="text-muted">Yardstick status page ({{version}}, {{date}})</h3>
         <nav>
           <ul class="nav nav-justified">
-            <li class="active"><a href="index.html">Home</a></li>
+            <li class="active"><a href="http://testresults.opnfv.org/reporting/index.html">Home</a></li>
             <li><a href="index-status-apex.html">Apex</a></li>
             <li><a href="index-status-compass.html">Compass</a></li>
             <li><a href="index-status-fuel.html">Fuel</a></li>
@@ -42,15 +89,15 @@
                     <tr>
                         <th width="40%">Scenario</th>
                         <th width="20%">Status</th>
-                        <th width="20%">Last 4 Iterations</th>
-                        <th width="20%">Last 10 Days</th>
+                        <th width="20%">Trend</th>
+                        <th width="10%">Last 4 Iterations</th>
+                        <th width="10%">Last 10 Days</th>
                     </tr>
                         {% for scenario,result in scenario_results.iteritems() -%}
                             <tr class="tr-ok">
                                 <td>{{scenario}}</td>
-                                <td>
-                                    <img src="../../img/gauge_{{ scenario_results[scenario].getStatus() }}.png">
-                                </td>
+                                <td><div id="gaugeScenario{{loop.index}}"></div></td>
+                                <td><div id="trend_svg{{loop.index}}"></div></td>
                                 <td>{{scenario_results[scenario].getFourDaysScore()}}</td>
                                 <td>{{scenario_results[scenario].getTenDaysScore()}}</td>
                             </tr>