Merge "Implement plan, qpi and metric constructors"

19 files changed, 165 insertions, 104 deletions

diff --git a/benchmarks/suite/compute.yaml b/benchmarks/suite/compute.yaml
index 0761a87b..197d5720 100644
--- a/benchmarks/suite/compute.yaml
+++ b/benchmarks/suite/compute.yaml
@@ -1,12 +1,12 @@
 QPI: compute
 description: sample performance index of computing
 
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
 
 section:
 - name: Integer
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: dhrystone
       workloads:
@@ -14,7 +14,7 @@ section:
         - multi_cpu
 - name: Floating
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: whetstone
       workloads:
@@ -22,7 +22,7 @@ section:
         - multi_cpu
 - name: Memory
   weight: 0.2
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: ramspeed
       workloads:
@@ -30,7 +30,7 @@ section:
           - float: [add, average, copy, scale, triad]
 - name: DPI
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: dpi
       workloads:
@@ -38,7 +38,7 @@ section:
         - pps
 - name: SSL
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: ssl
       workloads:
diff --git a/qtip/base/constant.py b/qtip/base/constant.py
index 187f0706..76481b47 100644
--- a/qtip/base/constant.py
+++ b/qtip/base/constant.py
@@ -8,8 +8,8 @@
 ##############################################################################
 
 
-class AlgoName(object):
-    """algorithm names"""
+class FormulaName(object):
+    """formula names"""
     ARITHMETIC_MEAN = 'arithmetic mean'
     WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
     GEOMETRIC_MEAN = 'geometric mean'
@@ -37,7 +37,7 @@ class PropName(object):
     # spec
     SECTIONS = 'sections'
     WEIGHT = 'weight'
-    ALGORITHM = 'algorithm'
+    FORMULA = 'formula'
     METRICS = 'metrics'
     WORKLOADS = 'workloads'
     # plan
diff --git a/qtip/base/error.py b/qtip/base/error.py
index d364c532..01a7f7a6 100644
--- a/qtip/base/error.py
+++ b/qtip/base/error.py
@@ -21,3 +21,16 @@ class NotFound(QtipError):
     def __init__(self, module, package='qtip'):
         self.package = package
         self.module = module
+
+
+class ToBeDoneError(QtipError):
+    """something still to be done"""
+    def __init__(self, method, module):
+        self.method = method
+        self.module = module
+
+
+def make_tbd(method, module='qtip'):
+    def tbd():
+        raise ToBeDoneError(method, module)
+    return tbd
diff --git a/qtip/cli/commands/cmd_plan.py b/qtip/cli/commands/cmd_plan.py
index 01bf8251..6f622e5a 100644
--- a/qtip/cli/commands/cmd_plan.py
+++ b/qtip/cli/commands/cmd_plan.py
@@ -9,7 +9,8 @@
 
 import click
 from prettytable import PrettyTable
-from qtip.runner.plan import Plan
+
+from qtip.loader.plan import Plan
 
 
 @click.group()
diff --git a/qtip/loader/base.py b/qtip/loader/base.py
index f7fcb669..2f5ab67a 100644
--- a/qtip/loader/base.py
+++ b/qtip/loader/base.py
@@ -20,13 +20,13 @@ ROOT_DIR = 'benchmarks'
 
 class BaseLoader(object):
     """Abstract class of QTIP benchmark loader"""
-    DEFAULT_DIR = '.'
+    RELATIVE_PATH = '.'
     _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
                         ROOT_DIR)]
 
     def __init__(self, name, paths=None):
         self._file = name
-        self._abspath = self._find(name, paths)
+        self._abspath = self._find(name, paths=paths)
 
         try:
             content = yaml.safe_load(file(self._abspath))
@@ -38,12 +38,11 @@ class BaseLoader(object):
             else path.splitext(name)[0]
         self.content = content
 
-    def _find(self, name, paths):
+    def _find(self, name, paths=None):
         """find a benchmark in searching paths"""
         paths = self._paths if paths is None else paths
-        name = path.join(self.DEFAULT_DIR, name)
         for p in paths:
-            abspath = path.join(p, name)
+            abspath = path.join(p, self.RELATIVE_PATH, name)
             if path.exists(abspath):
                 return abspath
         raise NotFound(name, paths)
@@ -52,7 +51,7 @@ class BaseLoader(object):
     def list_all(cls, paths=None):
         """list all available benchmarks"""
         paths = cls._paths if paths is None else paths
-        names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR))
+        names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
                                      for p in paths])
         for name in names:
             item = cls(name, paths=paths)
diff --git a/qtip/loader/metric.py b/qtip/loader/metric.py
index d6174e8f..8b6fa5d3 100644
--- a/qtip/loader/metric.py
+++ b/qtip/loader/metric.py
@@ -13,4 +13,4 @@ from base import BaseLoader
 class MetricSpec(BaseLoader):
     """metrics in QTIP are categorized by performance test tools, such as
     dhrystone, whetstone and etc"""
-    DEFAULT_DIR = 'metric'
+    RELATIVE_PATH = 'metric'
diff --git a/qtip/runner/plan.py b/qtip/loader/plan.py
index f6c1c3bb..cf517ea2 100644
--- a/qtip/runner/plan.py
+++ b/qtip/loader/plan.py
@@ -7,19 +7,21 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+
 from qtip.base.constant import PropName
-from qtip.runner.base import BaseRunner
+from qtip.loader.base import BaseLoader
 from qtip.loader.qpi import QPISpec
 
 
-class Plan(BaseRunner):
+class Plan(BaseLoader):
     """
     a benchmark plan is consist of configuration and a QPI list
     """
 
-    DEFAULT_DIR = 'plan'
+    RELATIVE_PATH = 'plan'
 
     def __init__(self, name, paths=None):
         super(Plan, self).__init__(name, paths)
+
         self.qpis = [QPISpec(qpi, paths=paths)
                      for qpi in self.content[PropName.QPIS]]
diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py
index 2972cbd7..cfa918c5 100644
--- a/qtip/loader/qpi.py
+++ b/qtip/loader/qpi.py
@@ -8,6 +8,10 @@
 ##############################################################################
 
 from base import BaseLoader
+from metric import MetricSpec
+
+from qtip.base.constant import PropName
+from qtip.utils.formula import Formula
 
 
 class QPISpec(BaseLoader):
@@ -15,4 +19,20 @@ class QPISpec(BaseLoader):
     a QPI specification defines how to calculate a performance index from
      collected metrics.
     """
-    DEFAULT_DIR = 'QPI'
+    RELATIVE_PATH = 'QPI'
+
+    def __init__(self, name, paths=None):
+        super(QPISpec, self).__init__(name, paths=paths)
+        content = self.content
+        self.formula = Formula(content[PropName.FORMULA])
+        self.sections = [Section(record, paths=paths)
+                         for record in content[PropName.SECTIONS]]
+
+
+class Section(object):
+    def __init__(self, content, paths=None):
+        self.name = content[PropName.NAME]
+        self.weight = content[PropName.WEIGHT]
+        self.formula = Formula(content[PropName.FORMULA])
+        self.metrics = [MetricSpec(record, paths=paths)
+                        for record in content[PropName.METRICS]]
diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py
index e69de29b..eab81156 100644
--- a/qtip/runner/__init__.py
+++ b/qtip/runner/__init__.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.stdout import StdoutCollector
+from qtip.driver.random import RandomDriver
+from qtip.reporter.console import ConsoleReporter
+
+
+class Runner(object):
+    def __init__(self, spec, config=None):
+        if config is None:
+            config = spec[PropName.CONFIG]
+
+        driver_name = config[PropName.DRIVER]
+        collector_name = config[PropName.COLLECTOR]
+        reporter_name = config[PropName.REPORTER]
+
+        # TODO(yujunz) dynamically load modules by name
+
+        if driver_name == 'random':
+            self.driver = RandomDriver()
+        else:
+            raise NotFound(driver_name, package=PkgName.DRIVER)
+
+        if collector_name == 'stdout':
+            self.collector = StdoutCollector()
+        else:
+            raise NotFound(collector_name,
+                           package=PkgName.COLLECTOR)
+
+        if reporter_name == 'console':
+            self.reporter = ConsoleReporter()
+        else:
+            raise NotFound(reporter_name,
+                           package=PkgName.REPORTER)
diff --git a/qtip/runner/base.py b/qtip/runner/base.py
deleted file mode 100644
index 07fba104..00000000
--- a/qtip/runner/base.py
+++ /dev/null
@@ -1,45 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.constant import PkgName, PropName
-from qtip.base.error import NotFound
-from qtip.collector.logfile import LogfileCollector
-from qtip.driver.sample import SampleDriver
-from qtip.loader.base import BaseLoader
-from qtip.reporter.console import ConsoleReporter
-
-
-class BaseRunner(BaseLoader):
-    def __init__(self, name, paths=None, config=None):
-        super(BaseRunner, self).__init__(name, paths=paths)
-        if config is None:
-            config = self.content[PropName.CONFIG]
-
-        driver_name = config[PropName.DRIVER]
-        collector_name = config[PropName.COLLECTOR]
-        reporter_name = config[PropName.REPORTER]
-
-        # TODO(yujunz) dynamically load modules by name
-
-        if driver_name == 'sample':
-            self.driver = SampleDriver()
-        else:
-            raise NotFound(driver_name, package=PkgName.DRIVER)
-
-        if collector_name == 'logfile':
-            self.collector = LogfileCollector()
-        else:
-            raise NotFound(collector_name,
-                           package=PkgName.COLLECTOR)
-
-        if reporter_name == 'console':
-            self.reporter = ConsoleReporter()
-        else:
-            raise NotFound(reporter_name,
-                           package=PkgName.REPORTER)
diff --git a/qtip/utils/formula.py b/qtip/utils/formula.py
new file mode 100644
index 00000000..cdfbae86
--- /dev/null
+++ b/qtip/utils/formula.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import numpy
+
+from qtip.base.error import make_tbd
+from qtip.base.constant import FormulaName
+
+
+MAPPING = {
+    FormulaName.ARITHMETIC_MEAN: numpy.mean,
+    FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average,
+    # TODO(yujunz) find or implement the method
+    FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__),
+    # TODO(yujunz) find or implement the method
+    FormulaName.WEIGHTED_GEOMETRIC_MEAN:
+        make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)}
+
+
+class Formula:
+    """calculate a score from give data"""
+    def __init__(self, name):
+        self.calculate = MAPPING[name]
diff --git a/requirements.txt b/requirements.txt
index cdaeef70..ec566b83 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ Flask==0.11.1
 Flask-RESTful==0.3.5
 flask-restful-swagger==0.19
 ansible==2.1.1.0
+numpy==1.11.3
diff --git a/tests/conftest.py b/tests/conftest.py
index f1ec91fb..7acb75e6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,9 +8,10 @@
 ##############################################################################
 
 from os import path
+
 import pytest
 
-from qtip.runner.plan import Plan
+from qtip.loader.plan import Plan
 
 
 @pytest.fixture(scope='session')
diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml
index 5ad8a4b6..e28efaf9 100644
--- a/tests/data/benchmarks/QPI/compute.yaml
+++ b/tests/data/benchmarks/QPI/compute.yaml
@@ -1,44 +1,29 @@
 title: compute
 description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
 sections:
 - name: Integer
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: dhrystone.yaml
-      workloads:
-        - single_cpu
-        - multi_cpu
+    - dhrystone.yaml
 - name: Float
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: dhrystone.yaml
-      workloads:
-        - single_cpu
-        - multi_cpu
+    - whetstone.yaml
 - name: Memory
   weight: 0.2
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: ramspeed.yaml
-      workloads:
-          - int: [add, average, copy, scale, triad]
-          - float: [add, average, copy, scale, triad]
+    - ramspeed.yaml
 - name: DPI
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: dpi.yaml
-      workloads:
-        - bps
-        - pps
+    - dpi.yaml
 - name: SSL
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: ssl.yaml
-      workloads:
-        - aes_128_cbc: [512, 1024, 2048, 4096]
-        - rsa_sig: [16, 64, 256, 1024, 8192]
+    - ssl.yaml
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml
index d75c6568..aa1097f4 100644
--- a/tests/data/benchmarks/QPI/fake-qpi.yaml
+++ b/tests/data/benchmarks/QPI/fake-qpi.yaml
@@ -1,9 +1,9 @@
 name: Fake QPI
 description: a fake QPI producing random result
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
 sections:
 - name: Fake Section
   weight: 0.5
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - fake_metric.yaml
+    - fake-metric.yaml
diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml
index b0d55ed2..220b7841 100644
--- a/tests/data/benchmarks/metric/dhrystone.yaml
+++ b/tests/data/benchmarks/metric/dhrystone.yaml
@@ -1,7 +1,9 @@
 name: dhrystone
 description: >
-  a synthetic computing benchmark program intended to be representative of
-  system (integer) programming
+  A synthetic computing benchmark program intended to be representative of
+  system (integer) programming.
+links:
+  - https://en.wikipedia.org/wiki/Dhrystone
 workloads:
   - single_cpu
   - multi_cpu
diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml
index d83680c4..448c9645 100644
--- a/tests/data/benchmarks/metric/whetstone.yaml
+++ b/tests/data/benchmarks/metric/whetstone.yaml
@@ -1,5 +1,10 @@
-name: dhrystone
-description: a synthetic benchmark for evaluating the performance of computers
+name: whetstone
+description: >
+  A synthetic benchmark for evaluating the performance of computers.
+  The Whetstone benchmark primarily measures the floating-point arithmetic
+  performance.
+links:
+  - https://en.wikipedia.org/wiki/Whetstone_(benchmark)
 workloads:
   - single_cpu
   - multi_cpu
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/loader/plan_test.py
index 7b3611d1..6aab5e8a 100644
--- a/tests/unit/runner/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -10,11 +10,14 @@
 import pytest
 
 from qtip.base.constant import PropName
-from qtip.runner.plan import Plan
+from qtip.loader.plan import Plan, QPISpec
 
 
 def test_init(plan):
     assert plan.name == 'fake plan'
+    assert isinstance(plan.content, dict)
+    for qpi in plan.qpis:
+        assert isinstance(qpi, QPISpec)
 
     with pytest.raises(TypeError) as excinfo:
         Plan()
@@ -36,3 +39,5 @@ def test_content(plan):
     content = plan.content
     assert PropName.NAME in content
     assert PropName.DESCRIPTION in content
+    assert PropName.CONFIG in content
+    assert PropName.QPIS in content
diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py
index c0d4b377..4b3fd4d0 100644
--- a/tests/unit/loader/qpi_test.py
+++ b/tests/unit/loader/qpi_test.py
@@ -9,7 +9,7 @@
 
 import pytest
 
-from qtip.base.constant import AlgoName, PropName
+from qtip.base.constant import FormulaName, PropName
 from qtip.loader.qpi import QPISpec
 
 QPI_SPEC = 'compute.yaml'
@@ -42,10 +42,10 @@ def test_list_all(benchmarks_root):
 def test_content(qpi_spec):
     content = qpi_spec.content
     assert PropName.DESCRIPTION in content
-    assert PropName.ALGORITHM in content
+    assert PropName.FORMULA in content
     assert PropName.SECTIONS in content
 
-    assert content[PropName.ALGORITHM] in AlgoName.__dict__.values()
+    assert content[PropName.FORMULA] in FormulaName.__dict__.values()
     sections = content[PropName.SECTIONS]
     assert isinstance(sections, list)
     for section in sections: