Refactor loader classes

- rename BaseLoader to YamlFileLoader as base class of QTIP specs loader
- create an abstract BaseLoader
- create FileLoader for logfile collector

Change-Id: I0c992cd847fc0dce4fdd73a13c1cdbc406c84532
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

11 files changed, 132 insertions, 69 deletions

diff --git a/qtip/base/constant.py b/qtip/base/constant.py
index 58cc0cc1..ddd07e9d 100644
--- a/qtip/base/constant.py
+++ b/qtip/base/constant.py
@@ -64,8 +64,8 @@ class PlanProp(BaseProp):
 class CollectorProp(BaseProp):
     LOGS = 'logs'
     FILENAME = 'filename'
-    PATTERNS = 'patterns'
-    MATCH = 'match'
+    GREP = 'grep'
+    REGEX = 'regex'
     CAPTURE = 'capture'
 
 
diff --git a/qtip/collector/base.py b/qtip/collector/base.py
index cd8fc797..2a25455c 100644
--- a/qtip/collector/base.py
+++ b/qtip/collector/base.py
@@ -10,3 +10,5 @@
 
 class BaseCollector(object):
     """performance metrics collector"""
+    def __init__(self, config):
+        self._config = config
diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py
index 6528ea9f..19780aaa 100644
--- a/qtip/collector/logfile.py
+++ b/qtip/collector/logfile.py
@@ -9,6 +9,30 @@
 
 from base import BaseCollector
 
+from qtip.base.constant import CollectorProp as CProp
+from qtip.loader.file import FileLoader
+
 
 class LogfileCollector(BaseCollector):
     """collect performance metrics from log files"""
+
+    def __init__(self, config, paths=None):
+        super(LogfileCollector, self).__init__(config)
+        self.loader = FileLoader('.', paths)
+
+    def collect(self):
+        captured = {}
+        for item in self._config[CProp.LOGS]:
+            captured.update(self._parse_log(item))
+        return captured
+
+    def _parse_log(self, log_item):
+        captured = {}
+        # TODO(yujunz) select parser by name
+        if CProp.GREP in log_item:
+            for rule in log_item[CProp.GREP]:
+                captured.update(self._grep(log_item[CProp.FILENAME], rule))
+        return captured
+
+    def _grep(self, filename, rule):
+        return {}
diff --git a/qtip/loader/base.py b/qtip/loader/base.py
index a0e5d031..abc254bf 100644
--- a/qtip/loader/base.py
+++ b/qtip/loader/base.py
@@ -7,57 +7,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from collections import defaultdict
-from itertools import chain
-from os import listdir
-from os import path
-import yaml
-
-from qtip.base.error import InvalidFormat, NotFound
-from qtip.base.constant import BaseProp
-
-
-ROOT_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir,
-                     'benchmarks')
-
 
 class BaseLoader(object):
-    """Abstract class of QTIP benchmark loader"""
-    RELATIVE_PATH = '.'
-    _paths = [ROOT_DIR]
-
-    def __init__(self, name, paths=None):
-        self._file = name
-        self._abspath = self._find(name, paths=paths)
-        content = defaultdict(lambda: None)
-
-        try:
-            content.update(yaml.safe_load(file(self._abspath)))
-        except yaml.YAMLError:
-            # TODO(yujunz) log yaml error
-            raise InvalidFormat(self._abspath)
-
-        self.name = content[BaseProp.NAME] or path.splitext(name)[0]
-        self.content = content
-
-    def _find(self, name, paths=None):
-        """find a benchmark in searching paths"""
-        paths = self._paths if paths is None else paths
-        for p in paths:
-            abspath = path.join(p, self.RELATIVE_PATH, name)
-            if path.exists(abspath):
-                return abspath
-        raise NotFound(name, paths)
-
-    @classmethod
-    def list_all(cls, paths=None):
-        """list all available benchmarks"""
-        paths = cls._paths if paths is None else paths
-        names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
-                                     for p in paths])
-        for name in names:
-            item = cls(name, paths=paths)
-            yield {
-                BaseProp.NAME: name,
-                BaseProp.ABSPATH: item._abspath,
-                BaseProp.CONTENT: item.content}
+    """Abstract class of QTIP loader"""
diff --git a/qtip/loader/file.py b/qtip/loader/file.py
new file mode 100644
index 00000000..00f94818
--- /dev/null
+++ b/qtip/loader/file.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from itertools import chain
+from os import listdir
+from os import path
+
+from qtip.base.constant import BaseProp
+from qtip.base.error import NotFound
+from qtip.loader.base import BaseLoader
+
+
+ROOT_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir,
+                     'benchmarks')
+
+
+class FileLoader(BaseLoader):
+    RELATIVE_PATH = '.'
+    _paths = [ROOT_DIR]
+
+    def __init__(self, name, paths=None):
+        self._file = name
+        self._abspath = self._find(name, paths=paths)
+
+    def _find(self, name, paths=None):
+        """find a specification in searching paths"""
+        paths = self._paths if paths is None else paths
+        for p in paths:
+            abspath = path.join(p, self.RELATIVE_PATH, name)
+            if path.exists(abspath):
+                return abspath
+        raise NotFound(name, paths)
+
+    @classmethod
+    def list_all(cls, paths=None):
+        """list all available specification"""
+        paths = cls._paths if paths is None else paths
+        names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
+                                     for p in paths])
+        for name in names:
+            item = cls(name, paths=paths)
+            yield {
+                BaseProp.NAME: name,
+                BaseProp.ABSPATH: item._abspath}
diff --git a/qtip/loader/metric.py b/qtip/loader/metric.py
index 8b6fa5d3..842fcdbf 100644
--- a/qtip/loader/metric.py
+++ b/qtip/loader/metric.py
@@ -7,10 +7,10 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from base import BaseLoader
+from yaml_file import YamlFileLoader
 
 
-class MetricSpec(BaseLoader):
+class MetricSpec(YamlFileLoader):
     """metrics in QTIP are categorized by performance test tools, such as
     dhrystone, whetstone and etc"""
     RELATIVE_PATH = 'metric'
diff --git a/qtip/loader/plan.py b/qtip/loader/plan.py
index c1ee00be..6f1764e2 100644
--- a/qtip/loader/plan.py
+++ b/qtip/loader/plan.py
@@ -9,11 +9,12 @@
 
 
 from qtip.base.constant import PlanProp
-from qtip.loader.base import BaseLoader
+from qtip.collector.logfile import LogfileCollector
+from qtip.loader.yaml_file import YamlFileLoader
 from qtip.loader.qpi import QPISpec
 
 
-class Plan(BaseLoader):
+class Plan(YamlFileLoader):
     """
     a benchmark plan is consist of configuration and a QPI list
     """
@@ -26,3 +27,7 @@ class Plan(BaseLoader):
         self.qpis = [QPISpec(qpi, paths=paths)
                      for qpi in self.content[PlanProp.QPIS]]
         self.info = self.content[PlanProp.INFO]
+        _config = self.content[PlanProp.CONFIG]
+
+        # TODO(yujunz) create collector by name
+        self.collector = LogfileCollector(_config[PlanProp.COLLECTOR], paths)
diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py
index ef6e065e..73da61e9 100644
--- a/qtip/loader/qpi.py
+++ b/qtip/loader/qpi.py
@@ -7,14 +7,14 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from base import BaseLoader
+from yaml_file import YamlFileLoader
 from metric import MetricSpec
 
 from qtip.base.constant import SpecProp
 from qtip.util.formula import Formula
 
 
-class QPISpec(BaseLoader):
+class QPISpec(YamlFileLoader):
     """
     a QPI specification defines how to calculate a performance index from
      collected metrics.
diff --git a/qtip/loader/yaml_file.py b/qtip/loader/yaml_file.py
new file mode 100644
index 00000000..f1cd4614
--- /dev/null
+++ b/qtip/loader/yaml_file.py
@@ -0,0 +1,33 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from collections import defaultdict
+from os import path
+import yaml
+
+from qtip.base.error import InvalidFormat
+from qtip.base.constant import BaseProp
+from qtip.loader.file import FileLoader
+
+
+class YamlFileLoader(FileLoader):
+    """load content from yaml file"""
+
+    def __init__(self, name, paths=None):
+        super(YamlFileLoader, self).__init__(name, paths)
+        content = defaultdict(lambda: None)
+
+        try:
+            content.update(yaml.safe_load(file(self._abspath)))
+        except yaml.YAMLError:
+            # TODO(yujunz) log yaml error
+            raise InvalidFormat(self._abspath)
+
+        self.name = content[BaseProp.NAME] or path.splitext(name)[0]
+        self.content = content
diff --git a/tests/data/opt/plan/doctor.yaml b/tests/data/benchmarks/plan/doctor.yaml
index 48b4c955..6c95077b 100644
--- a/tests/data/opt/plan/doctor.yaml
+++ b/tests/data/benchmarks/plan/doctor.yaml
@@ -10,24 +10,24 @@ config:
       logs:
         - filename: doctor_consumer.log
           # 2016-12-28 03:16:05,630 consumer.py 26 INFO   doctor consumer notified at 1482894965.63
-          patterns:
-            - match: 'doctor consumer notified at \d+(\.\d+)?$'
+          grep:
+            - regex: 'doctor consumer notified at \d+(\.\d+)?$'
               capture: notified consumer
         - filename: doctor_inspector.log
           # 2016-12-28 03:16:05,299 inspector.py 76 INFO   event posted at 1482894965.3
           # 2016-12-28 03:16:05,299 inspector.py 56 INFO   doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
           # 2016-12-28 03:16:05,506 inspector.py 66 INFO   doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
-          patterns:
-            - match: 'event posted at \d+(\.\d+)?$'
+          grep:
+            - regex: 'event posted at \d+(\.\d+)?$'
               capture: posted event
-            - match: 'doctor mark vm\(.*\) error at \d+(\.\d+)?$'
+            - regex: 'doctor mark vm\(.*\) error at \d+(\.\d+)?$'
               capture: marked VM error
-            - match: 'doctor mark host\(.*\) down at \d+(\.\d+)?$'
+            - regex: 'doctor mark host\(.*\) down at \d+(\.\d+)?$'
               capture: marked host down
         - filename: disable_network.log
           # doctor set host down at 1482894965.164096803
-          patterns:
-            - match: 'doctor set host down at \d+(\.\d+)?$'
+          grep:
+            - regex: 'doctor set host down at \d+(\.\d+)?$'
               capture: set host down
   reporter:
     name: console
diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py
index b57bcfb5..32837f8f 100644
--- a/tests/unit/loader/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -27,7 +27,7 @@ def test_init(plan):
 
 def test_list_all(benchmarks_root):
     plan_list = Plan.list_all(paths=[benchmarks_root])
-    assert len(list(plan_list)) is 1
+    assert len(list(plan_list)) is 2
     for desc in plan_list:
         assert PlanProp.NAME in desc
         assert PlanProp.CONTENT in desc