Merge "Add testing gating reporting page"

14 files changed, 495 insertions, 8 deletions

diff --git a/reporting/api/conf.py b/reporting/api/conf.py
index 5897d4f..e05c1fd 100644
--- a/reporting/api/conf.py
+++ b/reporting/api/conf.py
@@ -1 +1,5 @@
 base_url = 'http://testresults.opnfv.org/test/api/v1'
+
+versions = ['master', 'euphrates', 'fraser']
+
+period = 10
diff --git a/reporting/api/extension/__init__.py b/reporting/api/extension/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/reporting/api/extension/__init__.py
diff --git a/reporting/api/extension/client.py b/reporting/api/extension/client.py
new file mode 100644
index 0000000..03371fd
--- /dev/null
+++ b/reporting/api/extension/client.py
@@ -0,0 +1,15 @@
+from tornado.simple_httpclient import SimpleAsyncHTTPClient
+from tornado.log import gen_log
+
+
+class NoQueueTimeoutHTTPClient(SimpleAsyncHTTPClient):
+    def fetch_impl(self, request, callback):
+        key = object()
+
+        self.queue.append((key, request, callback))
+        self.waiting[key] = (request, callback, None)
+
+        self._process_queue()
+
+        if self.queue:
+            gen_log.debug("max_clients limit reached, request queued.")
diff --git a/reporting/api/handlers/scenarios.py b/reporting/api/handlers/scenarios.py
new file mode 100644
index 0000000..70447c7
--- /dev/null
+++ b/reporting/api/handlers/scenarios.py
@@ -0,0 +1,27 @@
+from tornado.web import asynchronous
+from tornado.gen import coroutine
+from tornado.escape import json_encode
+
+from api.handlers import BaseHandler
+from api.service.scenario import ScenarioTableResult
+
+
+class Result(BaseHandler):
+    @asynchronous
+    @coroutine
+    def get(self):
+        self._set_header()
+
+        scenario = self.get_argument('scenario', None)
+        version = self.get_argument('version', 'master')
+        installer = self.get_argument('installer', None)
+        iteration = int(self.get_argument('iteration', 10))
+
+        yield self._get_scenario_data(scenario, version, installer, iteration)
+
+    @coroutine
+    def _get_scenario_data(self, scenario, version, installer, iteration):
+        results = ScenarioTableResult(scenario, version, installer, iteration)
+        result = yield results.get()
+        self.write(json_encode(result))
+        self.finish()
diff --git a/reporting/api/server.py b/reporting/api/server.py
index e340b01..461e6d5 100644
--- a/reporting/api/server.py
+++ b/reporting/api/server.py
@@ -12,6 +12,7 @@ from tornado.options import define
 from tornado.options import options
 
 from api.urls import mappings
+from api.service.result import ResultCache
 
 define("port", default=8000, help="run on the given port", type=int)
 
@@ -20,7 +21,12 @@ def main():
     tornado.options.parse_command_line()
     application = tornado.web.Application(mappings)
     application.listen(options.port)
-    tornado.ioloop.IOLoop.current().start()
+
+    tornado.ioloop.PeriodicCallback(ResultCache.update, 1800000).start()
+
+    ioloop = tornado.ioloop.IOLoop.current()
+    ioloop.call_later(1000, ResultCache.update())
+    ioloop.start()
 
 
 if __name__ == "__main__":
diff --git a/reporting/api/service/__init__.py b/reporting/api/service/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/reporting/api/service/__init__.py
diff --git a/reporting/api/service/result.py b/reporting/api/service/result.py
new file mode 100644
index 0000000..fa6553c
--- /dev/null
+++ b/reporting/api/service/result.py
@@ -0,0 +1,90 @@
+import json
+import logging
+from collections import defaultdict
+
+from tornado.gen import coroutine
+from tornado.gen import Return
+from tornado.httpclient import AsyncHTTPClient
+
+from api import conf as consts
+from api.extension.client import NoQueueTimeoutHTTPClient
+
+LOG = logging.getLogger(__name__)
+AsyncHTTPClient.configure(NoQueueTimeoutHTTPClient)
+
+
+class Result(object):
+
+    def __init__(self):
+        self._url = '{}/results?period={}&version={}&page={}'
+        self._client = AsyncHTTPClient()
+        self._result = defaultdict(list)
+
+    @property
+    @coroutine
+    def result(self):
+        if not self._result:
+            yield self.update_results()
+        raise Return(self._result)
+
+    @coroutine
+    def update_results(self):
+        LOG.info('start update results')
+
+        for version in consts.versions:
+            yield self._update_version_result(version)
+
+        LOG.info('results update finished')
+
+    @coroutine
+    def _update_version_result(self, version):
+        total_page = yield self._get_total_page(version)
+
+        responses = yield self._fetch_results(version, total_page)
+
+        self._update_version_dict(version, responses)
+
+    @coroutine
+    def _fetch_results(self, version, total_page):
+        urls = [self._url.format(consts.base_url, consts.period, version, i)
+                for i in range(1, total_page + 1)]
+        responses = yield [self._client.fetch(url) for url in urls]
+        raise Return(responses)
+
+    @coroutine
+    def _get_total_page(self, version):
+        url = self._url.format(consts.base_url, consts.period, version, 1)
+        response = yield self._client.fetch(url)
+        raise Return(json.loads(response.body)['pagination']['total_pages'])
+
+    def _update_version_dict(self, version, responses):
+        for response in responses:
+            results = json.loads(response.body)['results']
+            for result in results:
+                data = {k: v for k, v in result.items() if k != 'details'}
+                self._result[version].append(data)
+
+
+class ResultCache(Result):
+
+    @classmethod
+    @coroutine
+    def update(cls):
+        cls._check()
+
+        yield cls.cache.update_results()
+
+    @classmethod
+    @coroutine
+    def get(cls, version):
+        cls._check()
+
+        result = yield cls.cache.result
+        raise Return(result[version])
+
+    @classmethod
+    def _check(cls):
+        try:
+            cls.cache
+        except AttributeError:
+            cls.cache = cls()
diff --git a/reporting/api/service/scenario.py b/reporting/api/service/scenario.py
new file mode 100644
index 0000000..f9e2a91
--- /dev/null
+++ b/reporting/api/service/scenario.py
@@ -0,0 +1,133 @@
+import abc
+from collections import defaultdict
+
+import six
+from tornado.gen import coroutine
+from tornado.gen import Return
+
+from api.service.result import ResultCache
+
+PROJECTS = ['fastdatastacks', 'barometer', 'sfc', 'sdnvpn', 'doctor', 'parser']
+
+
+def _set(key, llist):
+    return set(x[key] for x in llist)
+
+
+def _filter(key, value, llist):
+    return filter(lambda x: x[key] == value, llist)
+
+
+class ScenarioTableResult(object):
+
+    def __init__(self, scenario, version, installer, iteration):
+        self.scenario = scenario
+        self.version = version
+        self.installer = installer
+        self.iteration = iteration
+
+    @coroutine
+    def get(self):
+        results = yield ResultCache.get(self.version)
+        results = self._filter_result(results)
+        results = self._struct_result(results)
+
+        raise Return(results)
+
+    def _filter_result(self, results):
+        results = [x for x in results if x['build_tag']]
+        if self.installer:
+            results = _filter('installer', self.installer, results)
+        if self.scenario:
+            results = _filter('scenario', self.scenario, results)
+        return results
+
+    def _struct_result(self, results):
+
+        return {
+            s: self._struct_scenario(_filter('scenario', s, results))
+            for s in _set('scenario', results)
+        }
+
+    def _struct_scenario(self, data):
+        return sorted([
+            HandlerFacade.get_result(b, _filter('build_tag', b, data))
+            for b in _set('build_tag', data)
+        ], key=lambda x: x['date'], reverse=True)[:self.iteration]
+
+
+class HandlerFacade(object):
+
+    @classmethod
+    def get_result(cls, index, data):
+        if not data:
+            return {}
+
+        cls._change_name_to_functest(data)
+        data = cls._sort_by_start_date(data)
+
+        return {
+            'id': index,
+            'date': data[0]['start_date'],
+            'version': data[0]['version'],
+            'installer': data[0]['installer'],
+            'projects': cls._get_projects_data(data)
+        }
+
+    @classmethod
+    def _sort_by_start_date(cls, data):
+        return sorted(data, key=lambda x: x['start_date'])
+
+    @classmethod
+    def _get_projects_data(cls, data):
+
+        return {
+            p: HANDLER_MAP[p](_filter('project_name', p, data)).struct()
+            for p in _set('project_name', data)
+        }
+
+    @classmethod
+    def _change_name_to_functest(cls, data):
+        for ele in data:
+            if ele['project_name'] in PROJECTS:
+                ele['project_name'] = 'functest'
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ProjectHandler(object):
+
+    def __init__(self, data):
+        self.data = data
+
+    def struct(self):
+        return self._struct_project_data()
+
+    def _struct_project_data(self):
+        return {
+            'gating': self._gating()
+        }
+
+    @abc.abstractmethod
+    def _gating(self):
+        pass
+
+
+class DefaultHandler(ProjectHandler):
+
+    def _struct_project_data(self):
+        return {}
+
+    def _gating(self):
+        pass
+
+
+class FunctestHandler(ProjectHandler):
+
+    def _gating(self):
+        if all([x['criteria'] == 'PASS' for x in self.data]):
+            return 'PASS'
+        else:
+            return 'FAIL'
+
+
+HANDLER_MAP = defaultdict(lambda: DefaultHandler, functest=FunctestHandler)
diff --git a/reporting/api/urls.py b/reporting/api/urls.py
index a5228b2..34f61d8 100644
--- a/reporting/api/urls.py
+++ b/reporting/api/urls.py
@@ -9,6 +9,7 @@
 from api.handlers import landing
 from api.handlers import projects
 from api.handlers import testcases
+from api.handlers import scenarios
 
 mappings = [
     (r"/landing-page/filters", landing.FiltersHandler),
@@ -16,5 +17,7 @@ mappings = [
 
     (r"/projects-page/projects", projects.Projects),
     (r"/projects/([^/]+)/cases", testcases.TestCases),
-    (r"/projects/([^/]+)/cases/([^/]+)", testcases.TestCase)
+    (r"/projects/([^/]+)/cases/([^/]+)", testcases.TestCase),
+
+    (r"/scenarios/results", scenarios.Result)
 ]
diff --git a/reporting/pages/app/index.html b/reporting/pages/app/index.html
index 843a623..6ef1d02 100644
--- a/reporting/pages/app/index.html
+++ b/reporting/pages/app/index.html
@@ -87,6 +87,7 @@
     <script src="scripts/controllers/auth.controller.js"></script>
     <script src="scripts/controllers/admin.controller.js"></script>
     <script src="scripts/controllers/main.controller.js"></script>
+    <script src="scripts/controllers/gating.controller.js"></script>
     <script src="scripts/controllers/testvisual.controller.js"></script>
 
     <!-- endbuild -->
diff --git a/reporting/pages/app/scripts/config.router.js b/reporting/pages/app/scripts/config.router.js
index d3724b7..45a6997 100644
--- a/reporting/pages/app/scripts/config.router.js
+++ b/reporting/pages/app/scripts/config.router.js
@@ -17,11 +17,11 @@ angular.module('opnfvApp')
     ]).config(['$stateProvider', '$urlRouterProvider',
         function($stateProvider, $urlRouterProvider) {
 
-            $urlRouterProvider.otherwise('/landingpage/table');
+            $urlRouterProvider.otherwise('/home/gating');
 
             $stateProvider
-                .state('landingpage', {
-                    url: "/landingpage",
+                .state('home', {
+                    url: "/home",
                     controller: 'MainController',
                     templateUrl: "views/main.html",
                     data: { pageTitle: '首页', specialClass: 'landing-page' },
@@ -33,7 +33,7 @@ angular.module('opnfvApp')
                         }]
                     }
                 })
-                .state('landingpage.table', {
+                .state('home.table', {
                     url: "/table",
                     controller: 'TableController',
                     templateUrl: "views/commons/table.html",
@@ -47,6 +47,19 @@ angular.module('opnfvApp')
                         }]
                     }
                 })
+                .state('home.gating', {
+                    url: "/gating",
+                    controller: 'GatingController',
+                    templateUrl: "views/gating.html",
+                    data: { pageTitle: 'OPNFV Release Gating Page', specialClass: 'landing-page' },
+                    resolve: {
+                        controller: ['$ocLazyLoad', function($ocLazyLoad) {
+                            return $ocLazyLoad.load([
+
+                            ])
+                        }]
+                    }
+                })
                 .state('select', {
                     url: '/select',
                     templateUrl: "views/testcase.html",
diff --git a/reporting/pages/app/scripts/controllers/gating.controller.js b/reporting/pages/app/scripts/controllers/gating.controller.js
new file mode 100644
index 0000000..329f8e0
--- /dev/null
+++ b/reporting/pages/app/scripts/controllers/gating.controller.js
@@ -0,0 +1,119 @@
+'use strict';
+
+angular.module('opnfvApp')
+    .controller('GatingController', ['$scope', '$state', '$stateParams', 'TableFactory', function($scope, $state, $stateParams, TableFactory) {
+
+        init();
+
+        function init() {
+            $scope.goTest = goTest;
+            $scope.goLogin = goLogin;
+
+            $scope.scenarios = {};
+
+            $scope.scenarioList = _toSelectList(['all']);
+            $scope.versionList = _toSelectList(['master', 'euphrates', 'fraser']);
+            $scope.installerList = _toSelectList(['all', 'apex', 'compass', 'daisy', 'fuel', 'joid']);
+            $scope.iterationList = _toSelectList([10, 20, 30]);
+
+            $scope.selectScenario = 'all';
+            $scope.selectVersion = 'master';
+            $scope.selectInstaller = 'all';
+            $scope.selectIteration = 10;
+
+            $scope.ScenarioConfig = {
+                create: true,
+                valueField: 'title',
+                labelField: 'title',
+                delimiter: '|',
+                maxItems: 1,
+                placeholder: 'Scenario',
+                onChange: function(value) {
+                    $scope.selectScenario = value;
+                }
+            }
+
+            $scope.VersionConfig = {
+                create: true,
+                valueField: 'title',
+                labelField: 'title',
+                delimiter: '|',
+                maxItems: 1,
+                placeholder: 'Version',
+                onChange: function(value) {
+                    $scope.selectVersion = value;
+                    getScenarioResult();
+                }
+            }
+
+            $scope.InstallerConfig = {
+                create: true,
+                valueField: 'title',
+                labelField: 'title',
+                delimiter: '|',
+                maxItems: 1,
+                placeholder: 'Installer',
+                onChange: function(value) {
+                    $scope.selectInstaller = value;
+                    getScenarioResult();
+                }
+            }
+
+            $scope.IterationConfig = {
+                create: true,
+                valueField: 'title',
+                labelField: 'title',
+                delimiter: '|',
+                maxItems: 1,
+                placeholder: 'Iteration',
+                onChange: function(value) {
+                    $scope.selectIteration = value;
+                    getScenarioResult();
+                }
+            }
+            getScenarioResult();
+        }
+
+        function getScenarioResult(){
+            _getScenarioResult($scope.selectVersion, $scope.selectInstaller, $scope.selectIteration);
+        }
+
+        function _getScenarioResult(version, installer, iteration){
+            var data = {
+                'version': version,
+                'iteration': iteration
+            }
+
+            if(installer != 'all'){
+                data['installer'] = installer;
+            }
+
+            TableFactory.getScenarioResult().get(data).$promise.then(function(resp){
+                $scope.scenarios = resp;
+                _concat($scope.scenarioList, _toSelectList(Object.keys(resp)));
+            }, function(err){
+            });
+        }
+
+        function _concat(aList, bList){
+            angular.forEach(bList, function(ele){
+                aList.push(ele);
+            });
+        }
+
+        function _toSelectList(arr){
+            var tempList = [];
+            angular.forEach(arr, function(ele){
+                tempList.push({'title': ele});
+            });
+            return tempList;
+        }
+
+        function goTest() {
+            $state.go("select.selectTestCase");
+        }
+
+        function goLogin() {
+            $state.go("login");
+        }
+    }]);
diff --git a/reporting/pages/app/scripts/factory/table.factory.js b/reporting/pages/app/scripts/factory/table.factory.js
index e715c5c..a2c5c76 100644
--- a/reporting/pages/app/scripts/factory/table.factory.js
+++ b/reporting/pages/app/scripts/factory/table.factory.js
@@ -6,7 +6,7 @@
 angular.module('opnfvApp')
     .factory('TableFactory', function($resource, $rootScope, $http) {
 
-        var BASE_URL = 'http://testresults.opnfv.org/reporting2';
+        var BASE_URL = ' http://testresults.opnfv.org/testing';
         $.ajax({
           url: 'config.json',
           async: false,
@@ -15,11 +15,19 @@ angular.module('opnfvApp')
               BASE_URL = response.url;
           },
           error: function (response){
-              alert('fail to get api url, using default: http://testresults.opnfv.org/reporting2')
+              alert('fail to get api url, using default: http://testresults.opnfv.org/testing')
           }
         });
 
         return {
+            getScenarioResult: function() {
+                return $resource(BASE_URL + '/scenarios/results', {'scenario': '@scenario', 'version': '@version', 'installer': '@installer', 'iteration': '@iteration'}, {
+                    'get': {
+                        method: 'GET',
+
+                    }
+                });
+            },
             getFilter: function() {
                 return $resource(BASE_URL + '/landing-page/filters', {}, {
                     'get': {
diff --git a/reporting/pages/app/views/gating.html b/reporting/pages/app/views/gating.html
new file mode 100644
index 0000000..0215c87
--- /dev/null
+++ b/reporting/pages/app/views/gating.html
@@ -0,0 +1,68 @@
+<section class="container-tablesize">
+    <div class="row  border-bottom white-bg dashboard-header">
+        <div class="row">
+
+            <div class="ibox float-e-margins">
+                <div class="ibox-title">
+                    <h5>OPNFV Release Gating Reporting </h5>
+                </div>
+
+                <div class="ibox-content row">
+                    <div class="col-md-2" style="margin-top:5px;margin-right: 5px;">
+                        <selectize options="scenarioList" ng-model="selectScenario" config="ScenarioConfig"></selectize>
+                    </div>
+                    <div class="col-md-2" style="margin-top:5px;margin-right: 5px;">
+                        <selectize options="versionList" ng-model="selectVersion" config="VersionConfig"></selectize>
+                    </div>
+
+                    <div class="col-md-2" style="margin-top:5px;margin-right: 5px;">
+                        <selectize options="installerList" ng-model="selectInstaller" config="InstallerConfig"></selectize>
+
+                    </div>
+
+                    <div class="col-md-2" style="margin-top:5px;margin-right: 5px;">
+                        <selectize options="iterationList" ng-model="selectIteration" config="IterationConfig"></selectize>
+                    </div>
+                </div>
+
+                <div class="table-responsive">
+                    <table class="table table-bordered" id="table">
+                        <thead class="thead">
+                            <tr>
+                                <th>Scenario</th>
+                                <th>Date</th>
+                                <th>ID</th>
+                                <th>Version</th>
+                                <th>Installer</th>
+                                <th>Deployment</th>
+                                <th>Functest</th>
+                                <th>Yardstick</th>
+                            </tr>
+                        </thead>
+                        <tbody class="tbody" ng-show="selectScenario == 'all' || scenario == selectScenario" ng-repeat="(scenario,value) in scenarios">
+                            <tr ng-repeat="record in value">
+                                <td ng-if="$index == 0">{{ scenario }}</td>
+                                <td ng-if="$index != 0"></td>
+                                <td>{{ record.date }}</td>
+                                <td>{{ record.id }}</td>
+                                <td>{{ record.version }}</td>
+                                <td>{{ record.installer }}</td>
+                                <td>{{ record.projects.deployment.gating }}</td>
+                                <td>{{ record.projects.functest.gating }}</td>
+                                <td>{{ record.projects.yardstick.gating }}</td>
+                            </tr>
+                        </tbody>
+                    </table>
+
+                </div>
+
+            </div>
+        </div>
+    </div>
+
+</section>
+<style>
+.selectize-input {
+    width: 200px !important;
+}
+</style>