Collector: Add Multi-command collector.

This patch adds support to run following commands as 'collector'
1. collectd
2. PROX IRQ Latency
3. CROND (currently to collect ovs-port's CPU-mapping information)
4. Filebeat (send logs to elasticsearch)

Fixed pylint issues
Added description to control prox-runrapid. The use of 'stop' file.
Added a print in print_results.

JIRA: VSPERF-599

Change-Id: I583325ef14ddb0b292e8abddef1518d24b5e52d8
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>

4 files changed, 176 insertions, 0 deletions

diff --git a/conf/05_collector.conf b/conf/05_collector.conf
index a1bb41f8..b9974037 100644
--- a/conf/05_collector.conf
+++ b/conf/05_collector.conf
@@ -55,3 +55,14 @@ COLLECTD_INTERFACE_XKEYS = ['docker', 'lo']
 # Provide individual core-ids or range of core-ids.
 # The range is specified using '-'
 COLLECTD_INTELRDT_XKEYS = [ ]
+
+###############################################
+# Multi Command Collector Configurations
+###############################################
+MC_COLLECTD_CSV = '/tmp/csv/'
+MC_COLLECTD_CMD = '/opt/collectd/sbin/collectd'
+MC_PROX_HOME = '/home/opnfv/irq/'
+MC_PROX_CMD = './runrapid.py'
+MC_PROX_OUT = 'RUNirq.irq.log'
+MC_CRON_OUT = '/tmp/ovs-cores.log'
+MC_BEAT_CFILE = '/etc/filebeat/filebeat.yml'
diff --git a/docs/testing/user/configguide/tools.rst b/docs/testing/user/configguide/tools.rst
index 907e86d2..fbc0c71a 100644
--- a/docs/testing/user/configguide/tools.rst
+++ b/docs/testing/user/configguide/tools.rst
@@ -53,6 +53,9 @@ VSPERF assumes that collectd is installed and configured to send metrics over lo
 The metrics sent should be for the following categories: CPU, Processes, Interface,
 OVS, DPDK, Intel-RDT.
 
+For multicmd, apart from collectd, installation of PROX is also necessary.
+Installation steps for PROX can be found here - `DPPD-PROX <https://github.com/opnfv/samplevnf/tree/master/VNFs/DPPD-PROX>`_
+
 Configuration
 ^^^^^^^^^^^^^
 
@@ -79,6 +82,13 @@ The *collectd* configuration option includes:
 * ``COLLECTD_INTELRDT_KEYS`` - Interesting metrics from Intel-RDT
 * ``COLLECTD_INTERFACE_XKEYS`` - Metrics to exclude from Interface
 * ``COLLECTD_INTELRDT_XKEYS`` - Metrics to exclude from Intel-RDT
+* ``MC_COLLECTD_CSV`` - Path where collectd writes its metrics as CSV.
+* ``MC_COLLECTD_CMD`` - Path where Collectd is installed
+* ``MC_PROX_HOME`` - Path where PROX-IRQ is installed.
+* ``MC_PROX_CMD`` - Command to run PROX-IRQ
+* ``MC_PROX_OUT`` - Output file generated by PROX-IRQ stats collector.
+* ``MC_CRON_OUT`` - Output file path of the command run through CROND
+* ``MC_BEAT_CFILE`` - Filebeat configuration file path.
 
 
 .. _`Load Generators`:
diff --git a/tools/collectors/multicmd/__init__.py b/tools/collectors/multicmd/__init__.py
new file mode 100755
index 00000000..2ae2340f
--- /dev/null
+++ b/tools/collectors/multicmd/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2019 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Wrapper for multi-commands as a collector
+"""
diff --git a/tools/collectors/multicmd/multicmd.py b/tools/collectors/multicmd/multicmd.py
new file mode 100644
index 00000000..275a0693
--- /dev/null
+++ b/tools/collectors/multicmd/multicmd.py
@@ -0,0 +1,138 @@
+# Copyright 2019 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Collects information using various command line tools.
+"""
+
+#from tools.collectors.collector import collector
+import glob
+import logging
+import os
+from collections import OrderedDict
+from tools import tasks
+from tools.collectors.collector import collector
+from conf import settings
+
+class MultiCmd(collector.ICollector):
+    """ Multiple command-line controllers
+        collectd, prox, crond, filebeat
+    """
+    def __init__(self, results_dir, test_name):
+        """
+        initialize collectrs
+        """
+        self.prox_home = settings.getValue('MC_PROX_HOME')
+        self.collectd_cmd = settings.getValue('MC_COLLECTD_CMD')
+        self.collectd_csv = settings.getValue('MC_COLLECTD_CSV')
+        self.prox_out = settings.getValue('MC_PROX_OUT')
+        self.prox_cmd = settings.getValue('MC_PROX_CMD')
+        self.cron_out = settings.getValue('MC_CRON_OUT')
+        self.logger = logging.getLogger(__name__)
+        self.results_dir = results_dir
+        self.collectd_pid = 0
+        self.prox_pid = 0
+        self.cleanup_collectd_metrics()
+        self.logger.debug('%s', 'Multicmd data for '+ str(test_name))
+        # There should not be a file by name stop in prox_home folder
+        # Else Prox will start and stop immediately. This is a Hack to
+        # control prox-runrapid, which by default runs for specified duration.
+        filename = os.path.join(self.prox_home, 'stop')
+        if os.path.exists(filename):
+            tasks.run_task(['sudo', 'rm', filename],
+                           self.logger, 'deleting stop')
+        self.results = OrderedDict()
+
+    def cleanup_collectd_metrics(self):
+        """
+        Cleaup the old or archived metrics
+        """
+        for name in glob.glob(os.path.join(self.collectd_csv, '*')):
+            tasks.run_task(['sudo', 'rm', '-rf', name], self.logger,
+                           'Cleaning up Metrics', True)
+
+    def start(self):
+        # Command-1: Start Collectd
+        self.collectd_pid = tasks.run_background_task(
+            ['sudo', self.collectd_cmd],
+            self.logger, 'Staring Collectd')
+
+        # Command-2: Start PROX
+        working_dir = os.getcwd()
+        if os.path.exists(self.prox_home):
+            os.chdir(self.prox_home)
+            self.prox_pid = tasks.run_background_task(['sudo', self.prox_cmd,
+                                                       '--test', 'irq',
+                                                       '--env', 'irq'],
+                                                      self.logger,
+                                                      'Start PROX')
+        os.chdir(working_dir)
+        # Command-3: Start CROND
+        tasks.run_task(['sudo', 'systemctl', 'start', 'crond'],
+                       self.logger, 'Staring CROND', True)
+
+        # command-4: BEATS
+        tasks.run_task(['sudo', 'systemctl', 'start', 'filebeat'],
+                       self.logger, 'Starting BEATS', True)
+
+    def stop(self):
+        """
+        Stop All commands
+        """
+        # Command-1: COLLECTD
+        tasks.terminate_task_subtree(self.collectd_pid, logger=self.logger)
+        tasks.run_task(['sudo', 'pkill', '--signal', '2', 'collectd'],
+                       self.logger, 'Stopping Collectd', True)
+
+        # Backup the collectd-metrics for this test into a results folder
+        # results_dir = os.path.join(settings.getValue('RESULTS_PATH'), '/')
+        tasks.run_task(['sudo', 'cp', '-r', self.collectd_csv,
+                        self.results_dir], self.logger,
+                       'Copying Collectd Results File', True)
+        self.cleanup_collectd_metrics()
+
+        # Command-2: PROX
+        filename = os.path.join(self.prox_home, 'stop')
+        if os.path.exists(self.prox_home):
+            tasks.run_task(['sudo', 'touch', filename],
+                           self.logger, 'Stopping PROX', True)
+
+        outfile = os.path.join(self.prox_home, self.prox_out)
+        if os.path.exists(outfile):
+            tasks.run_task(['sudo', 'mv', outfile, self.results_dir],
+                           self.logger, 'Moving PROX-OUT file', True)
+
+        # Command-3: CROND
+        tasks.run_task(['sudo', 'systemctl', 'stop', 'crond'],
+                       self.logger, 'Stopping CROND', True)
+        if os.path.exists(self.cron_out):
+            tasks.run_task(['sudo', 'mv', self.cron_out, self.results_dir],
+                           self.logger, 'Move Cron Logs', True)
+
+        # Command-4: BEATS
+        tasks.run_task(['sudo', 'systemctl', 'stop', 'filebeat'],
+                       self.logger, 'Stopping BEATS', True)
+
+    def get_results(self):
+        """
+        Return results
+        """
+        return self.results
+
+    def print_results(self):
+        """
+        Print results
+        """
+        logging.info("Multicmd Output is not collected by VSPERF")
+        logging.info("Please refer to corresponding command's output")