Merge "project-ize kibana_dashboard"

15 files changed, 324 insertions, 21 deletions

diff --git a/dashboard/README.rst b/dashboard/README.rst
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/dashboard/README.rst
diff --git a/dashboard/dashboard/conf/config.py b/dashboard/dashboard/conf/config.py
index 48fed88..44cf797 100644
--- a/dashboard/dashboard/conf/config.py
+++ b/dashboard/dashboard/conf/config.py
@@ -22,7 +22,7 @@ class APIConfig:
     """
 
     def __init__(self):
-        self._default_config_location = "../etc/config.ini"
+        self._default_config_location = "/etc/dashboard/config.ini"
         self.es_url = 'http://localhost:9200'
         self.es_creds = None
         self.kibana_url = None
diff --git a/dashboard/dashboard/conf/testcases.py b/dashboard/dashboard/conf/testcases.py
index e120987..ff801b4 100644
--- a/dashboard/dashboard/conf/testcases.py
+++ b/dashboard/dashboard/conf/testcases.py
@@ -1,7 +1,7 @@
 import yaml
 
 
-with open('./functest/testcases.yaml') as f:
+with open('/etc/dashboard/testcases.yaml') as f:
     testcases_yaml = yaml.safe_load(f)
 f.close()
 
diff --git a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
index da7ccfc..651168b 100644
--- a/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
+++ b/dashboard/dashboard/elastic2kibana/dashboard_assembler.py
@@ -1,7 +1,7 @@
 import json
 
 import utility
-from common import elastic_access
+from dashboard.common import elastic_access
 
 
 class DashboardAssembler(object):
diff --git a/dashboard/dashboard/elastic2kibana/main.py b/dashboard/dashboard/elastic2kibana/main.py
index 4542c31..5af512e 100644
--- a/dashboard/dashboard/elastic2kibana/main.py
+++ b/dashboard/dashboard/elastic2kibana/main.py
@@ -4,10 +4,10 @@ import urlparse
 
 import argparse
 
-from common import elastic_access
-from common import logger_utils
-from conf import config
-from conf import testcases
+from dashboard.common import elastic_access
+from dashboard.common import logger_utils
+from dashboard.conf import config
+from dashboard.conf import testcases
 from dashboard_assembler import DashboardAssembler
 from visualization_assembler import VisualizationAssembler
 
diff --git a/dashboard/dashboard/elastic2kibana/utility.py b/dashboard/dashboard/elastic2kibana/utility.py
index dccd28a..55578bd 100644
--- a/dashboard/dashboard/elastic2kibana/utility.py
+++ b/dashboard/dashboard/elastic2kibana/utility.py
@@ -2,7 +2,7 @@ import json
 
 from jinja2 import Environment, PackageLoader
 
-env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
+env = Environment(loader=PackageLoader('dashboard', 'elastic2kibana/templates'))
 env.filters['jsonify'] = json.dumps
 
 
diff --git a/dashboard/dashboard/elastic2kibana/visualization_assembler.py b/dashboard/dashboard/elastic2kibana/visualization_assembler.py
index 1cb0ba8..d7e6e54 100644
--- a/dashboard/dashboard/elastic2kibana/visualization_assembler.py
+++ b/dashboard/dashboard/elastic2kibana/visualization_assembler.py
@@ -1,7 +1,7 @@
 import json
 
 import utility
-from common import elastic_access
+from dashboard.common import elastic_access
 
 
 class VisStateBuilder(object):
diff --git a/dashboard/dashboard/elastic2kibana_main.py b/dashboard/dashboard/elastic2kibana_main.py
deleted file mode 100644
index 3ec27cb..0000000
--- a/dashboard/dashboard/elastic2kibana_main.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from elastic2kibana.main import main
-
-if __name__ == '__main__':
-    main()
diff --git a/dashboard/dashboard/functest/format.py b/dashboard/dashboard/functest/format.py
new file mode 100644
index 0000000..ef485ba
--- /dev/null
+++ b/dashboard/dashboard/functest/format.py
@@ -0,0 +1,186 @@
+#! /usr/bin/env python
+
+
+def _convert_value(value):
+    return value if value != '' else 0
+
+
+def _convert_duration(duration):
+    if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration:
+        hours, minutes, seconds = duration.split(":")
+        hours = _convert_value(hours)
+        minutes = _convert_value(minutes)
+        seconds = _convert_value(seconds)
+        int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds)
+    else:
+        int_duration = duration
+    return int_duration
+
+
+def format_normal(testcase):
+    """
+    Look for these and leave any of those:
+        details.duration
+        details.tests
+        details.failures
+
+    If none are present, then return False
+    """
+    found = False
+    testcase_details = testcase['details']
+    fields = ['duration', 'tests', 'failures']
+    if isinstance(testcase_details, dict):
+        for key, value in testcase_details.items():
+            if key in fields:
+                found = True
+                if key == 'duration':
+                    testcase_details[key] = _convert_duration(value)
+            else:
+                del testcase_details[key]
+
+    if 'tests' in testcase_details and 'failures' in testcase_details:
+        testcase_tests = float(testcase_details['tests'])
+        testcase_failures = float(testcase_details['failures'])
+        if testcase_tests != 0:
+            testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests
+        else:
+            testcase_details['success_percentage'] = 0
+
+
+    return found
+
+
+def format_rally(testcase):
+    """
+    Structure:
+        details.[{summary.duration}]
+        details.[{summary.nb success}]
+        details.[{summary.nb tests}]
+
+    Find data for these fields
+        -> details.duration
+        -> details.tests
+        -> details.success_percentage
+    """
+    details = testcase['details']
+    summary = None
+    for item in details:
+        if 'summary' in item:
+            summary = item['summary']
+
+    if not summary:
+        return False
+
+    testcase['details'] = {
+        'duration': summary['duration'],
+        'tests': summary['nb tests'],
+        'success_percentage': summary['nb success']
+    }
+    return True
+
+
+def _get_statistics(orig_data, stat_fields, stat_values=None):
+    test_results = {}
+    for stat_data in orig_data:
+        for field in stat_fields:
+            stat_value = stat_data[field]
+            if stat_value in test_results:
+                test_results[stat_value] += 1
+            else:
+                test_results[stat_value] = 1
+
+    if stat_values is not None:
+        for stat_value in stat_values:
+            if stat_value not in test_results:
+                test_results[stat_value] = 0
+
+    return test_results
+
+
+def format_onos(testcase):
+    """
+    Structure:
+        details.FUNCvirNet.duration
+        details.FUNCvirNet.status.[{Case result}]
+        details.FUNCvirNetL3.duration
+        details.FUNCvirNetL3.status.[{Case result}]
+
+    Find data for these fields
+        -> details.FUNCvirNet.duration
+        -> details.FUNCvirNet.tests
+        -> details.FUNCvirNet.failures
+        -> details.FUNCvirNetL3.duration
+        -> details.FUNCvirNetL3.tests
+        -> details.FUNCvirNetL3.failures
+    """
+    testcase_details = testcase['details']
+
+    if 'FUNCvirNet' not in testcase_details or 'FUNCvirNetL3' not in testcase_details:
+        return False
+
+    funcvirnet_details = testcase_details['FUNCvirNet']['status']
+    funcvirnet_stats = _get_statistics(funcvirnet_details, ('Case result',), ('PASS', 'FAIL'))
+    funcvirnet_passed = funcvirnet_stats['PASS']
+    funcvirnet_failed = funcvirnet_stats['FAIL']
+    funcvirnet_all = funcvirnet_passed + funcvirnet_failed
+
+    funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status']
+    funcvirnetl3_stats = _get_statistics(funcvirnetl3_details, ('Case result',), ('PASS', 'FAIL'))
+    funcvirnetl3_passed = funcvirnetl3_stats['PASS']
+    funcvirnetl3_failed = funcvirnetl3_stats['FAIL']
+    funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed
+
+    testcase_details['FUNCvirNet'] = {
+        'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']),
+        'tests': funcvirnet_all,
+        'failures': funcvirnet_failed
+    }
+    testcase_details['FUNCvirNetL3'] = {
+        'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']),
+        'tests': funcvirnetl3_all,
+        'failures': funcvirnetl3_failed
+    }
+    return True
+
+
+def format_vims(testcase):
+    """
+    Structure:
+        details.sig_test.result.[{result}]
+        details.sig_test.duration
+        details.vIMS.duration
+        details.orchestrator.duration
+
+    Find data for these fields
+        -> details.sig_test.duration
+        -> details.sig_test.tests
+        -> details.sig_test.failures
+        -> details.sig_test.passed
+        -> details.sig_test.skipped
+        -> details.vIMS.duration
+        -> details.orchestrator.duration
+    """
+    testcase_details = testcase['details']
+    test_results = _get_statistics(testcase_details['sig_test']['result'],
+                                   ('result',),
+                                   ('Passed', 'Skipped', 'Failed'))
+    passed = test_results['Passed']
+    skipped = test_results['Skipped']
+    failures = test_results['Failed']
+    all_tests = passed + skipped + failures
+    testcase['details'] = {
+        'sig_test': {
+            'duration': testcase_details['sig_test']['duration'],
+            'tests': all_tests,
+            'failures': failures,
+            'passed': passed,
+            'skipped': skipped
+        },
+        'vIMS': {
+            'duration': testcase_details['vIMS']['duration']
+        },
+        'orchestrator': {
+            'duration': testcase_details['orchestrator']['duration']
+        }
+    }
+    return True
diff --git a/dashboard/dashboard/mongo2elastic/main.py b/dashboard/dashboard/mongo2elastic/main.py
index 8c7300b..c2a8d11 100644
--- a/dashboard/dashboard/mongo2elastic/main.py
+++ b/dashboard/dashboard/mongo2elastic/main.py
@@ -10,10 +10,11 @@ import uuid
 
 import argparse
 
-from common import logger_utils, elastic_access
-from conf import testcases
-from conf.config import APIConfig
-from mongo2elastic import format
+from dashboard.common import elastic_access
+from dashboard.common import logger_utils
+from dashboard.conf import testcases
+from dashboard.conf.config import APIConfig
+from dashboard.mongo2elastic import format
 
 logger = logger_utils.DashboardLogger('mongo2elastic').get
 
diff --git a/dashboard/dashboard/mongo2elastic_main.py b/dashboard/dashboard/mongo2elastic_main.py
deleted file mode 100644
index 141d8f3..0000000
--- a/dashboard/dashboard/mongo2elastic_main.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from mongo2elastic.main import main
-
-if __name__ == '__main__':
-    main()
diff --git a/dashboard/dashboard/qtip/format.py b/dashboard/dashboard/qtip/format.py
new file mode 100644
index 0000000..b78fa5b
--- /dev/null
+++ b/dashboard/dashboard/qtip/format.py
@@ -0,0 +1,19 @@
+#! /usr/bin/env python
+
+
+def format_qpi(testcase):
+    """
+    Look for these and leave any of those:
+        details.index
+
+    If none are present, then return False
+    """
+    details = testcase['details']
+    if 'index' not in details:
+        return False
+
+    for key, value in details.items():
+        if key != 'index':
+            del details[key]
+
+    return True
diff --git a/dashboard/install.sh b/dashboard/install.sh
new file mode 100755
index 0000000..9fd60d9
--- /dev/null
+++ b/dashboard/install.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+usage="
+Script to install dashboard automatically.
+This script should be run under root.
+
+usage:
+    bash $(basename "$0") [-h|--help] [-t <test_name>]
+
+where:
+    -h|--help         show this help text
+    -p|--project      project dashboard
+      <project_name>"
+
+# Parse parameters
+while [[ $# > 0 ]]
+    do
+    key="$1"
+    case $key in
+        -h|--help)
+            echo "$usage"
+            exit 0
+            shift
+        ;;
+        -p|--project)
+            PROJECT="$2"
+            shift
+        ;;
+        *)
+            echo "unknown option $1 $2"
+            exit 1
+        ;;
+    esac
+    shift # past argument or value
+done
+
+if [[ $(whoami) != "root" ]]; then
+    echo "Error: This script must be run as root!"
+    exit 1
+fi
+
+if [ -z ${PROJECT+x} ]; then
+    echo "project must be specified"
+    exit 1
+fi
+
+if [ $PROJECT != "functest" ] && [ $PROJECT != "qtip" ];then
+    echo "unsupported project $PROJECT"
+    exit 1
+fi
+
+cp -f dashboard/$PROJECT/format.py dashboard/mongo2elastic
+cp -f dashboard/$PROJECT/testcases.yaml etc/
+python setup.py install
diff --git a/dashboard/setup.cfg b/dashboard/setup.cfg
new file mode 100644
index 0000000..dd01358
--- /dev/null
+++ b/dashboard/setup.cfg
@@ -0,0 +1,43 @@
+[metadata]
+name = dashboard
+summary = Test Result Collector
+description-file =
+    README.rst
+author = SerenaFeng
+author-email = feng.xiaowei@zte.com.cn
+#home-page = http://www.opnfv.org/
+classifier =
+    Environment :: opnfv
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+
+[global]
+setup-hooks =
+    pbr.hooks.setup_hook
+
+[files]
+packages =
+    dashboard
+package_data =
+    dashboard =
+        elastic2kibana/templates/*.*
+data_files =
+    /etc/dashboard =
+        etc/config.ini
+        etc/testcases.yaml
+
+[entry_points]
+console_scripts =
+    dashboard_mongo2elastic = dashboard.mongo2elastic.main:main
+    dashboard_elastic2kibana = dashboard.elastic2kibana.main:main
+
+[egg_info]
+tag_build =
+tag_date = 0
+tag_svn_revision = 0
+
diff --git a/dashboard/setup.py b/dashboard/setup.py
new file mode 100644
index 0000000..59637a5
--- /dev/null
+++ b/dashboard/setup.py
@@ -0,0 +1,8 @@
+import setuptools
+
+__author__ = 'serena'
+
+
+setuptools.setup(
+    setup_requires=['pbr>=1.8'],
+    pbr=True)