Sysmetrics implementation update

New sysmetrics implementation is based on pidstat command line tool
from sysstat package. Old non-functional implementation was removed.
Reporting was refactored to generate report after each TC from values
already available in memory.
Following files were affected:
	modified:   conf/01_testcases.conf
	modified:   conf/02_vswitch.conf
	modified:   conf/05_collector.conf
	deleted:    core/collector_controller.py
	modified:   core/component_factory.py
	modified:   docs/NEWS.rst
	modified:   packages.txt
	modified:   requirements.txt
	modified:   testcases/testcase.py
	modified:   tools/collectors/collector/collector.py
	modified:   tools/collectors/sysmetrics/__init__.py
	deleted:    tools/collectors/sysmetrics/linuxmetrics.py
	new file:   tools/collectors/sysmetrics/pidstat.py
	modified:   tools/report/report.jinja
	modified:   tools/report/report.py
	modified:   tools/systeminfo.py
	modified:   vsperf

JIRA: VSPERF-67
Change-Id: I25a79f2afef405b9ac46ae85c18044af167a62a4
Signed-off-by: Martin Klozik (martinx.klozik@intel.com)
Reviewed-by: Billy O Mahony <billy.o.mahony@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Gurpreet Singh <gurpreet.singh@spirent.com>
Reviewed-by: Tv Rao <tv.rao@freescale.com>

3 files changed, 13 insertions, 22 deletions

diff --git a/conf/01_testcases.conf b/conf/01_testcases.conf
index cb4f9481..815e3a76 100755
--- a/conf/01_testcases.conf
+++ b/conf/01_testcases.conf
@@ -20,8 +20,6 @@
 # "Name": "phy2phy_burst",         # A human-readable string identifying the
 #                                  # test.
 # "Traffic Type": "rfc2544",       # One of the supported traffic types.
-# "Collector": "cpu, memory",      # Comma-separated list of Collectors to
-#                                  # be activated during this test.
 # "Deployment": "p2p",             # One of the supported deployment scenarios.
 # "Description": "Lorem ipsum..."  # Optional. A human-readable string
 #                                  # describing the test.
@@ -69,7 +67,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "phy2phy_tput",
         "Traffic Type": "rfc2544",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.Throughput.RFC2544.PacketLossRatio",
@@ -77,7 +74,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "back2back",
         "Traffic Type": "back2back",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
@@ -85,7 +81,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "phy2phy_tput_mod_vlan",
         "Traffic Type": "rfc2544",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "Frame Modification": "vlan",
         "biDirectional": "False",
@@ -94,7 +89,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "phy2phy_cont",
         "Traffic Type": "continuous",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "Description": "RFC2544 Phy2Phy Continuous Stream",
         "biDirectional": "True",
@@ -102,7 +96,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "phy2phy_scalability",
         "Traffic Type": "rfc2544",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.Scalability.RFC2544.0PacketLoss",
@@ -111,7 +104,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "pvp_tput",
         "Traffic Type": "rfc2544",
-        "Collector": "cpu",
         "Deployment": "pvp",
         "Description": "LTD.Throughput.RFC2544.PacketLossRatio",
         "biDirectional": "True",
@@ -119,7 +111,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "pvp_back2back",
         "Traffic Type": "back2back",
-        "Collector": "cpu",
         "Deployment": "pvp",
         "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
         "biDirectional": "True",
@@ -127,7 +118,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "phy2phy_cpu_load",
         "Traffic Type": "rfc2544",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.CPU.RFC2544.0PacketLoss",
@@ -141,7 +131,6 @@ PERFORMANCE_TESTS = [
     {
         "Name": "phy2phy_mem_load",
         "Traffic Type": "rfc2544",
-        "Collector": "cpu",
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.Memory.RFC2544.0PacketLoss",
diff --git a/conf/02_vswitch.conf b/conf/02_vswitch.conf
index e69511b2..73c42a9a 100644
--- a/conf/02_vswitch.conf
+++ b/conf/02_vswitch.conf
@@ -45,8 +45,8 @@ VSWITCH_VANILLA_KERNEL_MODULES = ['openvswitch']
 # directory where hugepages will be mounted on system init
 HUGEPAGE_DIR = '/dev/hugepages'
 
-# Sets OVS PMD core mask to 2 for CPU core 1 affinitization
-VSWITCH_PMD_CPU_MASK = '2'
+# Sets OVS PMDs core mask to 0x30 for affinitization to 5th and 6th CPU core
+VSWITCH_PMD_CPU_MASK = '0x30'
 VSWITCH_AFFINITIZATION_ON = 1
 
 VSWITCH_FLOW_TIMEOUT = '30000'
diff --git a/conf/05_collector.conf b/conf/05_collector.conf
index 4dcd8162..bda0ac8d 100644
--- a/conf/05_collector.conf
+++ b/conf/05_collector.conf
@@ -16,16 +16,18 @@
 # Collector configuration
 # ############################
 
-# ############################
-# Sysmetrics configuration
-# ############################
-
-COLLECTOR = 'LinuxMetrics'
+COLLECTOR = 'Pidstat'
 COLLECTOR_DIR = os.path.join(ROOT_DIR, 'tools/collectors')
 
-# the number of seconds between samples when calculating CPU percentage
-SYSMETRICS_LINUX_METRICS_CPU_SAMPLES_INTERVAL = 5
+# processes to be monitored by pidstat
+PIDSTAT_MONITOR = ['ovs-vswitchd', 'ovsdb-server', 'qemu-system-x86_64']
+
+# options which will be passed to pidstat
+PIDSTAT_OPTIONS = '-dur'
 
-# log file for sysmetrics
-LOG_FILE_SYS_METRICS = 'system-metrics.log'
+# sampling interval used by pidstat to collect statistics
+PIDSTAT_SAMPLE_INTERVAL = 1
 
+# prefix of pidstat's log file; separate log file is created
+# for each testcase in the directory with results
+LOG_FILE_PIDSTAT = 'pidstat'