Update functest to fix failing tests

* MCE log plugin was failing becaue it wasn't enables
  * Updated a check for kernel version to match Fraser
    compute nodes on Apex
* RDT was failing when virtual compute nodes were used
  * Add check to detect whether compute node is a VM
  * Skip RDT when running in a VM
* Gnocchi metrics were failing
  * increase the wait time between checking metrics
* CSV checks were failing on Hugepages
  * functest was assuming that tests were running on
    a system with at least two NUMA nodes
  * Only check for metrics on node0 of the system
* Report was showing FAIL even if a test was not executed
  * Update the report output to show SKIP instead

JIRA: BAROMETER-91
Change-Id: I4f70fc84fce80054a7aca2116357b0233f355f70
Signed-off-by: Emma Foley <emma.l.foley@intel.com>

3 files changed, 28 insertions, 9 deletions

diff --git a/baro_tests/collectd.py b/baro_tests/collectd.py
index 2682c2a0..188afab1 100644
--- a/baro_tests/collectd.py
+++ b/baro_tests/collectd.py
@@ -322,28 +322,28 @@ def _print_final_result_of_plugin(
             elif (id, out_plugin, plugin, False) in results:
                 print_line += ' FAIL   |'
             else:
-                print_line += ' NOT EX |'
+                print_line += ' SKIP   |'
         elif out_plugin == 'AODH':
             if (id, out_plugin, plugin, True) in results:
                 print_line += ' PASS   |'
             elif (id, out_plugin, plugin, False) in results:
                 print_line += ' FAIL   |'
             else:
-                print_line += ' FAIL   |'
+                print_line += ' SKIP   |'
         elif out_plugin == 'SNMP':
             if (id, out_plugin, plugin, True) in results:
                 print_line += ' PASS   |'
             elif (id, out_plugin, plugin, False) in results:
                 print_line += ' FAIL   |'
             else:
-                print_line += ' FAIL   |'
+                print_line += ' SKIP   |'
         elif out_plugin == 'CSV':
             if (id, out_plugin, plugin, True) in results:
                 print_line += ' PASS   |'
             elif (id, out_plugin, plugin, False) in results:
                 print_line += ' FAIL   |'
             else:
-                print_line += ' NOT EX |'
+                print_line += ' SKIP   |'
         else:
             print_line += ' SKIP   |'
     return print_line
@@ -468,6 +468,9 @@ def _exec_testcase(
         bridge for bridge in ovs_interfaces
         if bridge in ovs_configured_bridges]
     plugin_prerequisites = {
+        'intel_rdt': [(
+             conf.is_rdt_available(compute_node),
+             'RDT not avaialble on VMs')],
         'mcelog': [(
             conf.is_mcelog_installed(compute_node, 'mcelog'),
             'mcelog must be installed.')],
@@ -504,8 +507,7 @@ def _exec_testcase(
         'intel_rdt': [
             'intel_rdt-0-2'],
         'hugepages': [
-            'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb',
-            'hugepages-node1-2048Kb'],
+            'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb',],
         # 'ipmi': ['ipmi'],
         'mcelog': [
             'mcelog-SOCKET_0_CHANNEL_0_DIMM_any',
@@ -634,7 +636,7 @@ def mcelog_install():
     for node in nodes:
         if node.is_compute():
             centos_release = node.run_cmd('uname -r')
-            if '3.10.0-514.26.2.el7.x86_64' not in centos_release:
+            if centos_release not in ('3.10.0-514.26.2.el7.x86_64', '3.10.0-693.17.1.el7.x86_64'):
                 logger.info(
                     'Mcelog will NOT be enabled on node-{}.'
                     + ' Unsupported CentOS release found ({}).'.format(
diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py
index 6c67ee6a..2a4bc167 100644
--- a/baro_tests/config_server.py
+++ b/baro_tests/config_server.py
@@ -322,6 +322,17 @@ class ConfigServer(object):
                 else:
                     return 0
 
+    def is_rdt_available(self, compute):
+        """Check whether the compute node is a virtual machine."""
+        compute_name = compute.get_name()
+        nodes = get_apex_nodes()
+        for node in nodes:
+            if compute_name == node.get_dict()['name']:
+                stdout = node.run_cmd('cat /proc/cpuinfo | grep hypervisor')
+                if 'hypervisor' in stdout:
+                    return False
+        return True
+
     def is_libpqos_on_node(self, compute):
         """Check whether libpqos is present on compute node"""
 
@@ -532,7 +543,11 @@ class ConfigServer(object):
         timestamps1 = {}
         timestamps2 = {}
         nodes = get_apex_nodes()
-        sleep_time = plugin_interval + 2
+        if plugin_interval > 15:
+            sleep_time = plugin_interval*2
+        else:
+            sleep_time = 30
+
         for node in nodes:
             if node.is_controller():
                 self.__logger.info('Getting gnocchi metric list on {}' .format(
diff --git a/baro_tests/tests.py b/baro_tests/tests.py
index 580ee4d2..02eca90a 100644
--- a/baro_tests/tests.py
+++ b/baro_tests/tests.py
@@ -224,7 +224,9 @@ def test_csv_handles_plugin_data(
         + 'to interval...')
     for metric in plugin_metrics:
         logger.debug('{0} {1} {2} ... '.format(metric[0], metric[1], metric[2]))
-        if math.floor(metric[3] - metric[2]) > interval:
+        # When there's a small interval or many metrics, there may be a slight
+        # delay in writing the metrics. e.g. a gap of 1.* is okay for an interval of 1
+        if math.floor(metric[3] - metric[2]) > interval + 1:
             logger.error(
                 'Time of last two entries differ by '
                 + '{}, but interval is {}'.format(