Sysmetrics implementation update

New sysmetrics implementation is based on pidstat command line tool
from sysstat package. Old non-functional implementation was removed.
Reporting was refactored to generate report after each TC from values
already available in memory.
Following files were affected:
	modified:   conf/01_testcases.conf
	modified:   conf/02_vswitch.conf
	modified:   conf/05_collector.conf
	deleted:    core/collector_controller.py
	modified:   core/component_factory.py
	modified:   docs/NEWS.rst
	modified:   packages.txt
	modified:   requirements.txt
	modified:   testcases/testcase.py
	modified:   tools/collectors/collector/collector.py
	modified:   tools/collectors/sysmetrics/__init__.py
	deleted:    tools/collectors/sysmetrics/linuxmetrics.py
	new file:   tools/collectors/sysmetrics/pidstat.py
	modified:   tools/report/report.jinja
	modified:   tools/report/report.py
	modified:   tools/systeminfo.py
	modified:   vsperf

JIRA: VSPERF-67
Change-Id: I25a79f2afef405b9ac46ae85c18044af167a62a4
Signed-off-by: Martin Klozik (martinx.klozik@intel.com)
Reviewed-by: Billy O Mahony <billy.o.mahony@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Gurpreet Singh <gurpreet.singh@spirent.com>
Reviewed-by: Tv Rao <tv.rao@freescale.com>

2 files changed, 23 insertions, 28 deletions

diff --git a/tools/report/report.jinja b/tools/report/report.jinja
index 63d30fc0..491dbe99 100644
--- a/tools/report/report.jinja
+++ b/tools/report/report.jinja
@@ -88,12 +88,27 @@ A detailed summary of the main results for each test is outlined below.
 
 The following are the metrics obtained during this test:
 
-|              Metric            |             Result             |
+|             Metric             |             Result             |
 | ------------------------------ | ------------------------------ |
 {%- for item, value in test.result.items() %}
 | {{ "%-30s | %30s |"|format(item,value)}}
 {%- endfor %}
 
+##### Statistics collected
+
+The following system statistics were collected during testcase execution:
+{% for process in test.stats %}
+| --------------------------------------------------------------- |
+| Process: {{ "%-54s |"|format('_'.join(process.split('_')[:-1])) }}
+| ------------------------------ | ------------------------------ |
+|           Statistic            |             Value              |
+| ------------------------------ | ------------------------------ |
+{%- for item, value in test.stats[process].items() %}
+| {{ "%-30s | %30s |"|format(item,value)}}
+{%- endfor %}
+
+{% endfor %}
+
 ##### Anomalies
 
 No anomalies were detected during the course of this test.
diff --git a/tools/report/report.py b/tools/report/report.py
index 8d213297..806aecbc 100644
--- a/tools/report/report.py
+++ b/tools/report/report.py
@@ -21,10 +21,8 @@ Generate reports in format defined by X.
 import sys
 import os
 import jinja2
-import csv
 import logging
 
-from collections import OrderedDict
 from core.results.results_constants import ResultsConstants
 from conf import settings
 from tools import systeminfo
@@ -60,27 +58,7 @@ def _get_env():
     return env
 
 
-def _get_results(results_file):
-    """Get results from tests.
-
-    Get test results from a CSV file and return it as a list
-    of dictionaries for each row of data.
-
-    :param results_file: Path of the CSV results file
-
-    :returns: List of test results
-    """
-    with open(results_file, 'r') as csvfile:
-        reader = csv.reader(csvfile, delimiter=',')
-        result = []
-        res_head = next(reader)
-        for res_row in reader:
-            result.append(OrderedDict(zip(list(res_head), list(res_row))))
-
-    return result
-
-
-def generate(testcases, input_file):
+def generate(input_file, tc_results, tc_stats):
     """Generate actual report.
 
     Generate a Markdown-formatted file using results of tests and some
@@ -98,9 +76,9 @@ def generate(testcases, input_file):
 
     tests = []
     try:
-        for result in _get_results(input_file):
+        for result in tc_results:
             test_config = {}
-            for tc_conf in testcases:
+            for tc_conf in settings.getValue('PERFORMANCE_TESTS'):
                 if tc_conf['Name'] == result[ResultsConstants.ID]:
                     test_config = tc_conf
                     break
@@ -119,6 +97,7 @@ def generate(testcases, input_file):
                 'conf': test_config,
                 'result': result,
                 'env': _get_env(),
+                'stats': tc_stats
             })
 
         template_vars = {
@@ -131,12 +110,13 @@ def generate(testcases, input_file):
             logging.info('Test report written to "%s"', output_file)
 
     except KeyError:
-        logging.info("Report: Ignoring file (Wrongly defined columns): %s", (input_file))
+        logging.info("Report: Ignoring file (Wrongly defined columns): %s",
+                     (input_file))
         raise
     return output_file
 
 
 if __name__ == '__main__':
     settings.load_from_dir('conf')
-    OUT = generate(sys.argv[1])
+    OUT = generate(sys.argv[1], '', '')
     print('Test report written to "%s"...' % OUT)