Implement logfile collector and grep parser

JIRA: QTIP-207
JIRA: QTIP-208
Change-Id: Icc14d3097fb305e59df716636ef87504490c9d1b
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

10 files changed, 136 insertions, 37 deletions

diff --git a/tests/conftest.py b/tests/conftest.py
index 7acb75e6..32042f24 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,6 +12,7 @@ from os import path
 import pytest
 
 from qtip.loader.plan import Plan
+from qtip.loader.plan import PlanProp
 
 
 @pytest.fixture(scope='session')
@@ -26,4 +27,19 @@ def benchmarks_root(data_root):
 
 @pytest.fixture(scope='session')
 def plan(benchmarks_root):
-    return Plan('fake-plan.yaml', [benchmarks_root])
+    return Plan('doctor.yaml', [benchmarks_root])
+
+
+@pytest.fixture(scope='session')
+def plan_config(plan):
+    return plan.content[PlanProp.CONFIG]
+
+
+@pytest.fixture(scope='session')
+def collectors_config(plan_config):
+    return plan_config[PlanProp.COLLECTORS]
+
+
+@pytest.fixture(scope='session')
+def logfile_config(collectors_config):
+    return collectors_config[0]
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake_qpi.yaml
index aa1097f4..aa1097f4 100644
--- a/tests/data/benchmarks/QPI/fake-qpi.yaml
+++ b/tests/data/benchmarks/QPI/fake_qpi.yaml
diff --git a/tests/data/benchmarks/plan/doctor.yaml b/tests/data/benchmarks/plan/doctor.yaml
index f884c606..f8dcf08d 100644
--- a/tests/data/benchmarks/plan/doctor.yaml
+++ b/tests/data/benchmarks/plan/doctor.yaml
@@ -4,34 +4,30 @@ info:
   facility: local
   engineer: local
 config:
-  driver: sample
   collectors:
     - type: logfile
+      paths:
+        - '../../external/doctor-verify-apex-sample-master'
       logs:
         - filename: doctor_consumer.log
-          # 2016-12-28 03:16:05,630 consumer.py 26 INFO   doctor consumer notified at 1482894965.63
-          grep:
-            - regex: 'doctor consumer notified at \d+(\.\d+)?$'
-              capture: notified consumer
+          parsers:
+            - type: grep
+              regex: 'doctor consumer notified at (?P<notified>\d+(?:\.\d+)?)$'
         - filename: doctor_inspector.log
-          # 2016-12-28 03:16:05,299 inspector.py 76 INFO   event posted at 1482894965.3
-          # 2016-12-28 03:16:05,299 inspector.py 56 INFO   doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
-          # 2016-12-28 03:16:05,506 inspector.py 66 INFO   doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
-          grep:
-            - regex: 'event posted at \d+(\.\d+)?$'
-              capture: posted event
-            - regex: 'doctor mark vm\(.*\) error at \d+(\.\d+)?$'
-              capture: marked VM error
-            - regex: 'doctor mark host\(.*\) down at \d+(\.\d+)?$'
-              capture: marked host down
+          parsers:
+            - type: grep
+              regex: 'event posted at (?P<event_posted>\d+(?:\.\d+)?)$'
+            - type: grep
+              regex: 'doctor mark vm\(.*\) error at (?P<vm_error>\d+(?:\.\d+)?)$'
+            - type: grep
+              regex: 'doctor mark host\(.*\) down at (?P<host_down>\d+(?:\.\d+)?)$'
         - filename: disable_network.log
-          # doctor set host down at 1482894965.164096803
-          grep:
-            - regex: 'doctor set host down at \d+(\.\d+)?$'
-              capture: set host down
-  reporter:
-    name: console
-    # transform collected data into timeline
-    transformer: timeline
+          parsers:
+            - type: grep
+              regex: 'doctor set host down at (?P<network_down>\d+(?:\.\d+)?)$'
+  reporters:
+    - type: console
+      # transform collected data into timeline
+      transformer: timeline
 QPIs:
-  - fake-qpi.yaml
+  - fake_qpi.yaml
diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
deleted file mode 100644
index 511affd6..00000000
--- a/tests/data/benchmarks/plan/fake-plan.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-name: fake plan
-description: fake benchmark plan for demonstration and testing
-config:
-  facility: local
-  engineer: local
-  driver: sample
-  collectors: []
-  reporter: console
-QPIs:
-  - fake-qpi.yaml
diff --git a/tests/data/fake.log b/tests/data/fake.log
new file mode 100644
index 00000000..bab71e5a
--- /dev/null
+++ b/tests/data/fake.log
@@ -0,0 +1,9 @@
+Lorem ipsum dolor sit amet,
+consectetur adipiscing elit,
+sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+
+Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
+Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
diff --git a/tests/unit/collector/__init__.py b/tests/unit/collector/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/unit/collector/__init__.py
diff --git a/tests/unit/collector/base_test.py b/tests/unit/collector/base_test.py
new file mode 100644
index 00000000..17fe1af1
--- /dev/null
+++ b/tests/unit/collector/base_test.py
@@ -0,0 +1,18 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from qtip.loader.plan import load_collector
+from qtip.collector import CollectorProp as CProp
+
+
+def test_load_collector(collectors_config):
+    for c in collectors_config:
+        collector = load_collector(c[CProp.TYPE])
+        assert collector.TYPE == c[CProp.TYPE]
diff --git a/tests/unit/collector/grep_test.py b/tests/unit/collector/grep_test.py
new file mode 100644
index 00000000..e5d5f8c6
--- /dev/null
+++ b/tests/unit/collector/grep_test.py
@@ -0,0 +1,31 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+import pytest
+
+from qtip.collector.parser.grep import grep_in_file
+
+
+@pytest.fixture
+def logfile(data_root):
+    return os.path.join(data_root, 'fake.log')
+
+
+@pytest.mark.parametrize("regex,expected", [
+    ('not exist', []),
+    ('Lorem (\S+)', [{'groups': ('ipsum',), 'groupdict': {}}]),
+    ('nisi ut (?P<name>\S+)', [{'groups': ('aliquip',), 'groupdict': {'name': 'aliquip'}}])
+])
+def test_grep_in_file(logfile, regex, expected):
+    matches = grep_in_file(logfile, regex)
+    assert len(matches) == len(expected)
+    for i in range(len(matches)):
+        assert matches[i].groups() == expected[i]['groups']
+        assert matches[i].groupdict() == expected[i]['groupdict']
diff --git a/tests/unit/collector/logfile_test.py b/tests/unit/collector/logfile_test.py
new file mode 100644
index 00000000..a76aa3ee
--- /dev/null
+++ b/tests/unit/collector/logfile_test.py
@@ -0,0 +1,33 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.collector.logfile import LogfileCollector
+
+
+@pytest.fixture
+def logfile_collector(logfile_config, plan):
+    return LogfileCollector(logfile_config, plan)
+
+
+def test_run(logfile_collector):
+    collected = logfile_collector.run()
+    assert collected['groupdict'] == {
+        'event_posted': '1482894965.3',
+        'host_down': '1482894965.51',
+        'network_down': '1482894965.164096803',
+        'notified': '1482894965.63',
+        'vm_error': '1482894965.3'
+    }
+    assert list(collected['groups']) == ['1482894965.63',
+                                         '1482894965.3',
+                                         '1482894965.3',
+                                         '1482894965.51',
+                                         '1482894965.164096803']
diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py
index 81fd0bd3..4872b4cd 100644
--- a/tests/unit/loader/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -9,13 +9,15 @@
 
 import pytest
 
+from qtip.collector.logfile import LogfileCollector
+from qtip.loader.plan import load_collector
 from qtip.loader.plan import Plan
 from qtip.loader.plan import PlanProp
 from qtip.loader.plan import QPISpec
 
 
 def test_init(plan):
-    assert plan.name == 'fake plan'
+    assert plan.name == 'doctor performance profiling'
     assert isinstance(plan.content, dict)
     for qpi in plan.qpis:
         assert isinstance(qpi, QPISpec)
@@ -28,7 +30,7 @@ def test_init(plan):
 
 def test_list_all(benchmarks_root):
     plan_list = Plan.list_all(paths=[benchmarks_root])
-    assert len(list(plan_list)) is 2
+    assert len(list(plan_list)) is 1
     for desc in plan_list:
         assert PlanProp.NAME in desc
         assert PlanProp.CONTENT in desc
@@ -42,3 +44,7 @@ def test_content(plan):
     assert PlanProp.DESCRIPTION in content
     assert PlanProp.CONFIG in content
     assert PlanProp.QPIS in content
+
+
+def test_load_collector():
+    assert load_collector(LogfileCollector.TYPE) is LogfileCollector